?Unity + LeapMotion gesture recognition + hardware serial port interaction

Interactive test effect

1. Prepare the development environment

1. Download and install LeapMotion SDK (you can download it yourself)

2. The Unity editor I chose is 2018.4.32f1

2. Functional part

1. Determine the serial port number and baud rate to be connected. The program directly enters the code.

using KoboldCom;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using UnityEngine;
using UnityEngine.UI;

public class MyPorts : MonoBehaviour
{
    // Start is called before the first frame update
    Communicator communicator;
    public bool isHex = true;
    public Queue<String[]> DistanceData;
    public static MyPorts _instance;
    string[] config;
    //----
    public string[] configData;
    public int COM;
    public int BTL;

    public Text txtResult;

    private void Awake()
    {
        DistanceData = new Queue<string[]>();
        _instance = this;
    }
    void Start()
    {

#if UNITY_EDITOR
        config = File.ReadAllLines(Application.streamingAssetsPath + "/Config.txt");
#endif

#if UNITY_STANDALONE_WIN
        config = File.ReadAllLines(Application.dataPath + "/StreamingAssets/Config.txt");
#endif

        //config = File.ReadAllLines(Application.streamingAssetsPath + "/Config.txt");
        //Create communicator

        //-----
        GetConfigData();
        COM = int.Parse(configData[0]);
        BTL = int.Parse(configData[1]);

        communicator = new Communicator(new SerialPort(), null);
        openCom();

        StartCoroutine(QingQiu());

    }

    IEnumerator QingQiu()
    {
        while(true)
        {
            yield return null;

            if (LeapGestureTool.instance.isQin)
            {
                //Debug.Log(((int)LeapGestureTool.instance.distance/4).ToString("X2"));
                btnSend(((int)LeapGestureTool.instance.distance / 4).ToString("X2"));
            }
        }

    }

    public void openCom()
    {
        if (communicator == null)
        {
            // Debug.Log("communicator not initialized");
            txtResult.text = "communicator is not initialized";
            return;
        }
        SerialPortSetting sps = new SerialPortSetting();
        sps.Baudrate = BTL; //Baud rate
        sps.Port = COM;//Serial port Com3
        if (communicator.Com.Open(sps))
        {
            //Debug.Log("Serial port opened successfully");
            txtResult.text = "Serial port opened successfully";
        }
        else
        {
            txtResult.text = "Serial port opening failed";
            // Debug.Log("Failed to open serial port");
        }
    }
    private void OnDestroy()
    {
        closeCom();
    }
    public void closeCom()
    {
        if (communicator == null)
        {
            Debug.Log("communicator is not initialized");
            return;
        }
        communicator.Com.Close();
    }

    public void btnSend(string senddata)
    {
        sendData(strToToHexByte(senddata));
        //sendData(Encoding.UTF8.GetBytes(senddata));
    }

    void Update()
    {
        if (communicator != null & amp; & amp; communicator.Com.IsOpen & amp; & amp; communicator.Com.BytesToRead > 0)
        {
            if(isHex)
            {
                int count = communicator.Com.BytesToRead;
                byte[] buffer = new byte[count];
                communicator.Com.Read(buffer, 0, count);
                OnRawDataReceived(buffer);
            }
            else
            {
                string data = communicator.Com.ReadLine();
                Debug.Log("data: " + data);
                if (data.Length > 30)
                {
                    DistanceData.Enqueue(data.Split(' '));
                }
            }
        }
    }


    private void OnRawDataReceived(byte[] bytes)
    {
        StringBuilder builder = new StringBuilder();
        foreach (byte b in bytes)
        {
            builder.Append(b.ToString("X2") + " ");
        }
        //Debug.Log("Received serial port data:" + builder.ToString());
        //if (builder.ToString().CompareTo("99 02 0D 0A ") == 0)
        //{
        // Debug.Log("Right");
        //}
        //else
        //{
        // Debug.Log("There is a discrepancy in the received data");
        //}
    }

    public void sendData(byte[] d)
    {
        if (communicator == null)
        {
            Debug.Log("communicator is not initialized");
            return;
        }
        if (d == null || d.Length == 0)
        {
            Debug.Log("Sending data is empty");
            return;
        }
        try
        {
            communicator.Com.Write(d, 0, d.Length);
        }
        catch (Exception e)
        {
            print("Sending failed");
        }
    }

    public string[] GetConfigData()
    {
        configData = new string[config.Length];
        for (int i = 0; i < configData.Length; i + + )
        {
            configData[i] = config[i].Substring(config[i].IndexOf(':') + 1);
        }
        return configData;
    }




    private static byte[] strToToHexByte(string hexString)
    {
        hexString = hexString.Replace(" ", "");
        if ((hexString.Length % 2) != 0)
            hexString + = " ";
        byte[] returnBytes = new byte[hexString.Length / 2];
        for (int i = 0; i < returnBytes.Length; i + + )
            returnBytes[i] = Convert.ToByte(hexString.Substring(i * 2, 2), 16);
        return returnBytes;
    }
}

2.LeapMotion code

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Leap.Unity;
using Leap;

public class LeapGestureTool : MonoBehaviour
{
    public static LeapGestureTool instance;
    private Controller leapController;
    public Camera leapCamera;

    public bool isQin = false;
    public float distance = 0;

    private void Awake()
    {
        instance = this;
    }

    void Start()
    {
        //Create a LeapMotion controller instance
        leapController = new Controller();
    }

    void Update()
    {
        // Check if the Leap Motion controller is connected
        if (!leapController.IsConnected)
        {
            Debug.Log("Leap Motion is not connected");
            return;
        }

        // Get the latest frame data
        Frame frame = leapController.Frame();

        // Get the first detected hand
        if (frame.Hands.Count > 0)
        {
            isQin = true;

            Hand hand = frame.Hands[0];

            // Get the hand position
            Vector3 handPosition = hand.PalmPosition.ToVector3();

            // Get the camera's position
            Vector3 cameraPosition = leapCamera.transform.position;

            // Calculate the distance between the human hand and the camera
             distance = Vector3.Distance(handPosition, cameraPosition);
            
            Debug.Log("The distance between the human hand and the Leap Motion camera is: " + distance);
        }
        else
        {
            isQin = false;
            //Debug.Log("Hand is not in the detection range");
        }
    }

}