Switching to version 0.9.4 of Body Tracking SDK. New joints for hands…

…, confidence level for joints, CPU only mode (very slow!)

K4AdotNet.Samples.Core.BodyTrackingSpeed/ProcessingParameters.cs

@@ -8,6 +8,7 @@ internal sealed class ProcessingParameters
         public const string MKV_FILE_EXTENSION = ".mkv";
 
         public static readonly string MkvPathDescription = "Path to MKV file";
+        public static readonly string CpuOnlyModeDescription = "CPU/GPU mode (C - use only CPU, G - use GPU, default - G)";
         public static readonly string ImplementationDescription = "Optional implementation type (S - single thread, P - pop in background, E - enqueue in background, default - S)";
         public static readonly string StartTimeDescription = "Optional start time of video interval in seconds (default - beginning of recording)";
         public static readonly string EndTimeDescription = "Optional end time of video interval in seconds (default - end of recording)";
@@ -26,6 +27,7 @@ public static bool IsValueLikeToMkvFilePath(string value)
         }
 
         public string MkvPath { get; private set; }
+        public bool CpuOnlyMode { get; private set; }
         public ProcessingImplementation Implementation { get; private set; }
         public TimeSpan? StartTime { get; private set; }
         public TimeSpan? EndTime { get; private set; }
@@ -68,6 +70,32 @@ public bool TrySetMkvPath(string value, out string message)
             return true;
         }
 
+        public bool TrySetCpuOnlyMode(string value, out string message)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                CpuOnlyMode = false;
+                message = null;
+                return true;
+            }
+
+            value = value.Trim().ToLowerInvariant();
+            switch (value)
+            {
+                case "c":
+                    CpuOnlyMode = true;
+                    message = null;
+                    return true;
+                case "g":
+                    CpuOnlyMode = false;
+                    message = null;
+                    return true;
+            }
+
+            message = $"Invalid value. Expected 'C' or 'G'.";
+            return false;
+        }
+
         public bool TrySetImplementation(string value, out string message)
         {
             if (string.IsNullOrWhiteSpace(value))

K4AdotNet.Samples.Core.BodyTrackingSpeed/Processor.cs

@@ -30,7 +30,9 @@ protected Processor(ProcessingParameters processingParameters)
             playback.GetCalibration(out calibration);
             if (processingParameters.StartTime.HasValue)
                 Seek(processingParameters.StartTime.Value);
-            tracker = new BodyTracking.Tracker(ref calibration);
+            var config = BodyTracking.TrackerConfiguration.Default;
+            config.CpuOnlyMode = processingParameters.CpuOnlyMode;
+            tracker = new BodyTracking.Tracker(ref calibration, config);
         }
 
         public virtual void Dispose()

K4AdotNet.Samples.Core.BodyTrackingSpeed/Program.cs

@@ -62,6 +62,8 @@ private static ProcessingParameters AskProcessingParameters()
             var parameters = new ProcessingParameters();
             if (!AskParameter(ProcessingParameters.MkvPathDescription, parameters.TrySetMkvPath))
                 return null;
+            if (!AskParameter(ProcessingParameters.CpuOnlyModeDescription, parameters.TrySetCpuOnlyMode))
+                return null;
             if (!AskParameter(ProcessingParameters.ImplementationDescription, parameters.TrySetImplementation))
                 return null;
             if (!AskParameter(ProcessingParameters.StartTimeDescription, parameters.TrySetStartTime))
@@ -100,6 +102,11 @@ private static ProcessingParameters ParseCommandLineArguments(string[] args)
                 var arg = args[i].Trim();
                 switch (arg.ToLowerInvariant())
                 {
+                    case "-m":
+                    case "--mode":
+                        if (!ParseArgument(args, ref i, parameters.TrySetCpuOnlyMode))
+                            return null;
+                        break;
                     case "-i":
                     case "--implementation":
                         if (!ParseArgument(args, ref i, parameters.TrySetImplementation))
@@ -168,6 +175,7 @@ private static void PrintHowToUse()
             Console.WriteLine("where: ");
             Console.WriteLine("  <mkvFile> - " + ProcessingParameters.MkvPathDescription);
             Console.WriteLine("  options:");
+            Console.WriteLine("    -m, --mode c|g\t\t" + ProcessingParameters.StartTimeDescription);
             Console.WriteLine("    -i, --implementation s|p|e\t\t" + ProcessingParameters.StartTimeDescription);
             Console.WriteLine("    -s, --startTime <time>\t\t" + ProcessingParameters.StartTimeDescription);
             Console.WriteLine("    -e, --endTime <time>\t\t" + ProcessingParameters.EndTimeDescription);

K4AdotNet.Samples.Wpf.BodyTracker/BackgroundTrackingLoop.cs

@@ -11,9 +11,9 @@ internal sealed class BackgroundTrackingLoop : IDisposable
         private readonly Thread backgroundThread;
         private volatile bool isRunning;
 
-        public BackgroundTrackingLoop(ref Calibration calibration, SensorOrientation sensorOrientation, float smoothingFactor)
+        public BackgroundTrackingLoop(ref Calibration calibration, bool cpuOnlyMode, SensorOrientation sensorOrientation, float smoothingFactor)
         {
-            var config = new TrackerConfiguration { SensorOrientation = sensorOrientation };
+            var config = new TrackerConfiguration { SensorOrientation = sensorOrientation, CpuOnlyMode = cpuOnlyMode };
             tracker = new Tracker(ref calibration, config) { TemporalSmoothingFactor = smoothingFactor };
             isRunning = true;
             backgroundThread = new Thread(BackgroundLoop) { IsBackground = true };

K4AdotNet.Samples.Wpf.BodyTracker/MainModel.cs

@@ -44,7 +44,7 @@ public void Playback()
                             disableColor: DisableColor,
                             disableDepth: false,
                             doNotPlayFasterThanOriginalFps: DoNotPlayFasterThanOriginalFps);
-                        var viewModel = new TrackerModel(app, readingLoop, SensorOrientation, SmoothingFactor);
+                        var viewModel = new TrackerModel(app, readingLoop, CpuOnlyMode, SensorOrientation, SmoothingFactor);
                         app.ShowWindowForModel(viewModel);
                     }
                 }
@@ -102,7 +102,7 @@ public void OpenDevice()
                 using (app.IndicateWaiting())
                 {
                     var readingLoop = BackgroundReadingLoop.CreateForDevice(device, DepthMode, ColorResolution, FrameRate);
-                    var viewModel = new TrackerModel(app, readingLoop, SensorOrientation, SmoothingFactor);
+                    var viewModel = new TrackerModel(app, readingLoop, CpuOnlyMode, SensorOrientation, SmoothingFactor);
                     app.ShowWindowForModel(viewModel);
                 }
             }
@@ -117,6 +117,13 @@ public void OpenDevice()
 
         #region Settings
 
+        public bool CpuOnlyMode
+        {
+            get => cpuOnlyMode;
+            set => SetPropertyValue(ref cpuOnlyMode, value, nameof(CpuOnlyMode));
+        }
+        private bool cpuOnlyMode;
+
         public float SmoothingFactor
         {
             get => smoothingFactor;

K4AdotNet.Samples.Wpf.BodyTracker/MainWindow.xaml

@@ -80,21 +80,23 @@
                         <RowDefinition/>
                         <RowDefinition/>
                         <RowDefinition/>
+                        <RowDefinition/>
                     </Grid.RowDefinitions>
-                    <Label Grid.Row="0" Grid.Column="0" Margin="3">Orientation:</Label>
-                    <ComboBox Grid.Row="0" Grid.Column="1" Margin="3"
+                    <CheckBox Grid.Row="0" Grid.Column="1" Margin="3" IsChecked="{Binding CpuOnlyMode}">CPU only mode</CheckBox>
+                    <Label Grid.Row="1" Grid.Column="0" Margin="3">Orientation:</Label>
+                    <ComboBox Grid.Row="1" Grid.Column="1" Margin="3"
                             ItemsSource="{Binding SensorOrientations}"
                             DisplayMemberPath="Value"
                             SelectedValuePath="Key"
                             SelectedValue="{Binding SensorOrientation}"/>
-                    <Label Grid.Row="1" Grid.Column="0" Margin="3,0">Smoothing:</Label>
-                    <Slider Grid.Row="1" Grid.Column="1" Margin="3,0" VerticalAlignment="Bottom"
+                    <Label Grid.Row="2" Grid.Column="0" Margin="3,0">Smoothing:</Label>
+                    <Slider Grid.Row="2" Grid.Column="1" Margin="3,0" VerticalAlignment="Bottom"
                             Minimum="0" Maximum="1" SmallChange="0.05" LargeChange="0.25"
                             TickPlacement="BottomRight"
                             Value="{Binding SmoothingFactor}"/>
-                    <TextBlock Grid.Row="2" Grid.Column="1" FontSize="14" FontWeight="Light" TextAlignment="Left" VerticalAlignment="Top">&lt; low</TextBlock>
-                    <TextBlock Grid.Row="2" Grid.Column="1" FontSize="14" FontWeight="Light" TextAlignment="Right" VerticalAlignment="Top">high &gt;</TextBlock>
-                    <TextBlock Grid.Row="2" Grid.Column="1" FontSize="14" TextAlignment="Center" VerticalAlignment="Top"><Run Text="{Binding SmoothingFactor, Mode=OneWay, StringFormat=0.00}"/></TextBlock>
+                    <TextBlock Grid.Row="3" Grid.Column="1" FontSize="14" FontWeight="Light" TextAlignment="Left" VerticalAlignment="Top">&lt; low</TextBlock>
+                    <TextBlock Grid.Row="3" Grid.Column="1" FontSize="14" FontWeight="Light" TextAlignment="Right" VerticalAlignment="Top">high &gt;</TextBlock>
+                    <TextBlock Grid.Row="3" Grid.Column="1" FontSize="14" TextAlignment="Center" VerticalAlignment="Top"><Run Text="{Binding SmoothingFactor, Mode=OneWay, StringFormat=0.00}"/></TextBlock>
                 </Grid>
             </StackPanel>
         </GroupBox>

K4AdotNet.Samples.Wpf.BodyTracker/SkeletonVisualizer.cs

@@ -33,7 +33,9 @@ public SkeletonVisualizer(Dispatcher dispatcher, int widthPixels, int heightPixe
             JointCircleRadius = heightPixels / 70;
             JointBorder = new Pen(Brushes.Black, JointCircleRadius / 4);
             BonePen = new Pen(Brushes.White, JointCircleRadius * 2 / 3);
-            JointFill = Brushes.LightGray;
+            JointFill = Brushes.LightGreen;
+            JointFillLow = Brushes.Yellow;
+            JointFillNone = Brushes.OrangeRed;
         }
 
         /// <summary>
@@ -49,6 +51,12 @@ public SkeletonVisualizer(Dispatcher dispatcher, int widthPixels, int heightPixe
         /// <summary>Visualization setting: brush to fill joint circle.</summary>
         public Brush JointFill { get; set; }
 
+        /// <summary>Visualization setting: brush to fill joint circle for Low confidence.</summary>
+        public Brush JointFillLow { get; set; }
+
+        /// <summary>Visualization setting: brush to fill joint circle for None confidence.</summary>
+        public Brush JointFillNone { get; set; }
+
         /// <summary>Visualization setting: radius of joint circle.</summary>
         public double JointCircleRadius { get; set; }
 
@@ -127,18 +135,30 @@ private void DrawJoints(DrawingContext dc, Skeleton skeleton)
         {
             foreach (var jointType in JointTypes.All)
             {
-                var point2D = ProjectJointToImage(skeleton[jointType]);
+                var joint = skeleton[jointType];
+                var point2D = ProjectJointToImage(joint);
                 if (point2D.HasValue)
                 {
                     var radius = JointCircleRadius;
                     // smaller radius for face features (eyes, ears, nose)
                     if (jointType.IsFaceFeature())
                         radius /= 2;
-                    dc.DrawEllipse(JointFill, JointBorder, point2D.Value, radius, radius);
+                    var fillBrush = JointConfidenceLevelToToBrush(joint.ConfidenceLevel);
+                    dc.DrawEllipse(fillBrush, JointBorder, point2D.Value, radius, radius);
                 }
             }
         }
 
+        private Brush JointConfidenceLevelToToBrush(JointConfidenceLevel confidenceLevel)
+        {
+            switch (confidenceLevel)
+            {
+                case JointConfidenceLevel.None: return JointFillNone;
+                case JointConfidenceLevel.Low: return JointFillLow;
+                default: return JointFill;
+            }
+        }
+
         private readonly Dispatcher dispatcher;
         private readonly int widthPixels;
         private readonly int heightPixels;

K4AdotNet.Samples.Wpf.BodyTracker/TrackerModel.cs

@@ -32,12 +32,12 @@ public TrackerModel()
             DepthColumnWidth = ColorColumnWidth = new GridLength(1, GridUnitType.Star);
         }
 
-        public TrackerModel(IApp app, BackgroundReadingLoop readingLoop, SensorOrientation sensorOrientation, float smoothingFactor)
+        public TrackerModel(IApp app, BackgroundReadingLoop readingLoop, bool cpuOnlyMode, SensorOrientation sensorOrientation, float smoothingFactor)
             : base(app)
         {
             // try to create tracking loop first
             readingLoop.GetCalibration(out calibration);
-            trackingLoop = new BackgroundTrackingLoop(ref calibration, sensorOrientation, smoothingFactor);
+            trackingLoop = new BackgroundTrackingLoop(ref calibration, cpuOnlyMode, sensorOrientation, smoothingFactor);
             trackingLoop.BodyFrameReady += TrackingLoop_BodyFrameReady;
             trackingLoop.Failed += BackgroundLoop_Failed;
 

K4AdotNet/BodyTracking/Joint.cs

@@ -5,8 +5,9 @@ namespace K4AdotNet.BodyTracking
     // Defined in k4abttypes.h:
     // typedef struct _k4abt_joint_t
     // {
-    //     k4a_float3_t position;    /**< The position of the joint specified in millimeters*/
-    //     k4a_quaternion_t orientation; /**< The orientation of the joint specified in normalized quaternion*/
+    //     k4a_float3_t position;
+    //     k4a_quaternion_t orientation;
+    //     k4abt_joint_confidence_level_t confidence_level;
     // } k4abt_joint_t;
     //
     /// <summary>Structure to define a single joint.</summary>
@@ -22,5 +23,8 @@ public struct Joint
 
         /// <summary>The orientation of the joint specified in normalized quaternion.</summary>
         public Quaternion Orientation;
+
+        /// <summary>The confidence level of the joint.</summary>
+        public JointConfidenceLevel ConfidenceLevel;
     }
 }

K4AdotNet/BodyTracking/JointConfidenceLevel.cs

@@ -0,0 +1,28 @@
+namespace K4AdotNet.BodyTracking
+{
+    // Defined in k4abttypes.h:
+    // typedef enum
+    // {
+    //     K4ABT_JOINT_CONFIDENCE_NONE = 0,
+    //     K4ABT_JOINT_CONFIDENCE_LOW = 1,
+    //     K4ABT_JOINT_CONFIDENCE_MEDIUM = 2,
+    //     K4ABT_JOINT_CONFIDENCE_HIGH = 3,
+    //     K4ABT_JOINT_CONFIDENCE_LEVELS_COUNT = 4,
+    // } k4abt_joint_confidence_level_t;
+    //
+    /// <summary>This enumeration specifies the joint confidence level.</summary>
+    public enum JointConfidenceLevel
+    {
+        /// <summary>The joint is out of range (too far from depth camera).</summary>
+        None = 0,
+
+        /// <summary>The joint is not observed (likely due to occlusion), predicted joint pose.</summary>
+        Low = 1,
+
+        /// <summary>Medium confidence in joint pose. Current SDK will only provide joints up to this confidence level.</summary>
+        Medium = 2,
+
+        /// <summary>High confidence in joint pose. Placeholder for future SDK.</summary>
+        High = 3,
+    }
+}

K4AdotNet/BodyTracking/JointType.cs

@@ -11,10 +11,16 @@
     //     K4ABT_JOINT_SHOULDER_LEFT,
     //     K4ABT_JOINT_ELBOW_LEFT,
     //     K4ABT_JOINT_WRIST_LEFT,
+    //     K4ABT_JOINT_HAND_LEFT,
+    //     K4ABT_JOINT_HANDTIP_LEFT,
+    //     K4ABT_JOINT_THUMB_LEFT,
     //     K4ABT_JOINT_CLAVICLE_RIGHT,
     //     K4ABT_JOINT_SHOULDER_RIGHT,
     //     K4ABT_JOINT_ELBOW_RIGHT,
     //     K4ABT_JOINT_WRIST_RIGHT,
+    //     K4ABT_JOINT_HAND_RIGHT,
+    //     K4ABT_JOINT_HANDTIP_RIGHT,
+    //     K4ABT_JOINT_THUMB_RIGHT,
     //     K4ABT_JOINT_HIP_LEFT,
     //     K4ABT_JOINT_KNEE_LEFT,
     //     K4ABT_JOINT_ANKLE_LEFT,
@@ -62,6 +68,15 @@ public enum JointType
         /// <summary>Left wrist joint.</summary>
         WristLeft,
 
+        /// <summary>Left hand joint.</summary>
+        HandLeft,
+
+        /// <summary>Tip of left hand joint.</summary>
+        HandTipLeft,
+
+        /// <summary>Left thumb joint.</summary>
+        ThumbLeft,
+
         /// <summary>Right clavicle joint.</summary>
         ClavicleRight,
 
@@ -74,6 +89,15 @@ public enum JointType
         /// <summary>Right wrist joint.</summary>
         WristRight,
 
+        /// <summary>Right hand joint.</summary>
+        HandRight,
+
+        /// <summary>Tip of right hand joint.</summary>
+        HandTipRight,
+
+        /// <summary>Right thumb joint.</summary>
+        ThumbRight,
+
         /// <summary>Left hip joint.</summary>
         HipLeft,