azurekinect - How to implement Gesture swipe left right up down for Azure Kinect?

Question

I am trying to implement 4 swipe gesture(to left, right, up, down) for azure kinect, I reference the article that implement gesture for old kinect. And I modified the code for the azure kinect. (https://pterneas.com/2014/01/27/implementing-kinect-gestures/)

Also, I use this library for azure kinect (https://github.com/bibigone/k4a.net)

Also, I realized that it work not very good, I added cooldown between gestures. How can I do better for the gesture detection?

using K4AdotNet.BodyTracking;
using K4AdotNet.Samples.Unity;
using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public enum GesturePartResult
{
    Failed,
    Succeeded
}

public enum GestureType
{
    Left,
    Right,
    Up,
    Down,
}


public interface IGestureSegment
{
    GesturePartResult Update(Skeleton skeleton);
}



public class HandRightOfElbowSegment : IGestureSegment
{
    public GesturePartResult Update(Skeleton skeleton)
    {
        //Debug.Log($"{skeleton[JointType.HandRight].ConfidenceLevel}");
        // Hand above elbow
        if (-skeleton[JointType.HandRight].PositionMm.Y >
            -skeleton[JointType.ElbowRight].PositionMm.Y)
        {
            //Debug.Log($"hand above");

            // Hand right of elbow
            if (-skeleton[JointType.HandRight].PositionMm.X >
                -skeleton[JointType.ElbowRight].PositionMm.X)
            {
                //Debug.Log($"hand right");
                return GesturePartResult.Succeeded;
            }
        }

        // Hand dropped
        return GesturePartResult.Failed;
    }
}

public class HandLeftOfElbowSegment : IGestureSegment
{
    public GesturePartResult Update(Skeleton skeleton)
    {
        // Hand above elbow
        if (-skeleton[JointType.HandRight].PositionMm.Y >
            -skeleton[JointType.ElbowRight].PositionMm.Y)
        {
            // Hand left of elbow
            if (-skeleton[JointType.HandRight].PositionMm.X <
                -skeleton[JointType.ElbowRight].PositionMm.X)
            {
                return GesturePartResult.Succeeded;
            }
        }

        // Hand dropped
        return GesturePartResult.Failed;
    }
}

public class ElbowAboveShoulderSegment : IGestureSegment
{
    
    public GesturePartResult Update(Skeleton skeleton)
    {
        if (-skeleton[JointType.ElbowRight].PositionMm.Y > -skeleton[JointType.ShoulderRight].PositionMm.Y)
        {
            return GesturePartResult.Succeeded;
        }

        return GesturePartResult.Failed;
    }
}

public class ElbowBelowShoulderSegment : IGestureSegment
{
    public GesturePartResult Update(Skeleton skeleton)
    {
        if (-skeleton[JointType.ElbowRight].PositionMm.Y < -skeleton[JointType.ShoulderRight].PositionMm.Y)
        {
            return GesturePartResult.Succeeded;
        }

        return GesturePartResult.Failed;
    }
}

public class Gesture
{
    readonly int WINDOW_SIZE = 30;

    IGestureSegment[] _segments;

    int _currentSegment = 0;
    int _frameCount = 0;

    public event EventHandler GestureRecognized;
    public GestureType type;
    public GameObject gameObject;

    public Gesture(GestureType type)
    {
        HandRightOfElbowSegment handRightOfElbow = new HandRightOfElbowSegment();
        HandLeftOfElbowSegment handLeftOfElbow = new HandLeftOfElbowSegment();
        ElbowAboveShoulderSegment elbowAboveShoulder = new ElbowAboveShoulderSegment();
        ElbowBelowShoulderSegment elbowBelowShoulder = new ElbowBelowShoulderSegment();

        this.type = type;

        switch (type)
        {
            case GestureType.Left:
                _segments = new IGestureSegment[]
                {
                    handRightOfElbow,
                    //handRightOfElbow,
                    //handLeftOfElbow,
                    handLeftOfElbow,
                    //handRightOfHead,
                    //handLeftOfHead
                };
                break;
            case GestureType.Right:
                _segments = new IGestureSegment[]
                {
                    handLeftOfElbow,
                    //handLeftOfElbow,
                    //handRightOfElbow,
                    handRightOfElbow,
                };
                break;
            case GestureType.Up:
                _segments = new IGestureSegment[]
{
                    elbowBelowShoulder,
                    elbowAboveShoulder,
};
                break;
            case GestureType.Down:
                _segments = new IGestureSegment[]
{
                    elbowAboveShoulder,
                    elbowBelowShoulder,
};
                break;
            default:
                break;
        }

    }


    public void Update(Skeleton skeleton)
    {
        //Debug.Log($"current segment {_currentSegment}");

        GesturePartResult result = _segments[_currentSegment].Update(skeleton);

        if (result == GesturePartResult.Succeeded)
        {
            //Debug.Log($"Succeeded");
            if (_currentSegment + 1 < _segments.Length)
            {
                _currentSegment++;
                _frameCount = 0;
            }
            else
            {
                //Debug.Log($"hello");

                if (GestureRecognized != null)
                {
                    GestureRecognized(this, new EventArgs());
                    Reset();
                }
            }
        }
        else if (_frameCount == WINDOW_SIZE)
        {
            //Debug.Log($"Failed {_frameCount}");
            Reset();
        }
        else
        {
            _frameCount++;
        }
    }

    public void Reset()
    {
        _currentSegment = 0;
        _frameCount = 0;
    }
}



public class GestureDetect : MonoBehaviour
{
    // gesture
    private Gesture left = new Gesture(GestureType.Left);
    private Gesture right = new Gesture(GestureType.Right);
    private Gesture up = new Gesture(GestureType.Up);
    private Gesture down = new Gesture(GestureType.Down);

    // cooldown
    [SerializeField]
    private float cooldown = .5f;
    [SerializeField]
    private float currentCooldown = 0f;


    // menu
    [SerializeField]
    private Menu menu;


    public GameObject hand;

    private void OnEnable()
    {
        var skeletonProvider = FindObjectOfType<SkeletonProvider>();
        if (skeletonProvider != null)
        {
            skeletonProvider.SkeletonUpdated += OnSkeletonUpdate;
        }

        left.GestureRecognized += OnGestureRecognized;
        right.GestureRecognized += OnGestureRecognized;
        up.GestureRecognized += OnGestureRecognized;
        down.GestureRecognized += OnGestureRecognized;
    }

    private void OnDisable()
    {
        var skeletonProvider = FindObjectOfType<SkeletonProvider>();
        if (skeletonProvider != null)
        {
            skeletonProvider.SkeletonUpdated -= OnSkeletonUpdate;
        }
    }


    private void OnSkeletonUpdate(object sender, SkeletonEventArgs e)
    {
        if (e.Skeleton == null) return;

        var joint = e.Skeleton.Value.WristRight;
        hand.transform.position = new Vector3(-joint.PositionMm.X, -joint.PositionMm.Y, 0) * 0.001f * 10;


        if (currentCooldown > 0f) return;
        left.Update(e.Skeleton.Value);

        if (currentCooldown > 0f) return;
        right.Update(e.Skeleton.Value);

        if (currentCooldown > 0f) return;
        up.Update(e.Skeleton.Value);

        if (currentCooldown > 0f) return;
        down.Update(e.Skeleton.Value);
    }

    private void OnGestureRecognized(object sender, EventArgs e)
    {
        Debug.Log($"Gesture recognized {((Gesture)sender).type} {currentCooldown}");

        left.Reset();
        right.Reset();
        up.Reset();
        down.Reset();

        currentCooldown = cooldown;


        switch (((Gesture)sender).type)
        {
            case GestureType.Left:
                // left
                break;
            case GestureType.Right:
                // right
                break;
            case GestureType.Up:
                // up
                break;
            case GestureType.Down:
                // down
                break;
        }
    }
    private void Update()
    {
        if (currentCooldown > 0f)
        {
            currentCooldown -= Time.deltaTime;
        }
    }
}

question from:https://stackoverflow.com/questions/65951880/how-to-implement-gesture-swipe-left-right-up-down-for-azure-kinect

Answer 1

Waitting for answers