Getting Started with EmguCv and Unity3D - Face detection using webcam

In this we will create a simple project to turn cube around by detecting face using a webcam

Tools I have used.

1. Unity3D 5.3.3 personal edition
2. EmguCv - emgucv-windesktop 3.1.0.2282
3. A webcam

Getting Started

1. Setup EmguCv for Unity3D by creating a new project (follow the below post if you have no idea how to set it up. http://sharewhatulearn.blogspot.in/2016/04/setting-up-emgucv-for-unity3d.html)
2. Once you are done with all the basic setup go to Unity3D and create a Cube under GameObjects-> 3D Objects->Cube
3. Set the cube x,y,z to 0 
4. Select the main Camera and set its x=0, y=0, and Z= -3
5. Next Create a Empty Game object GameObject-> Create Empty and name it as "FaceDetect"
6. Create a new folder under Assets and name it as Scripts.
7. Under Scripts folder Create a new C# Script file named "CameraScript"
8. Again create another C# script file named "FaceDetectScript"
9. Now select the "FaceDetect" Game Object and attach the "FaceDetectScript" to it.
10. Now Select the Main Camera and attach the "CameraScript" to it.
11. Copy the below script and paste it to CameraScriptusing UnityEngine;
    using System.Collections;
    
    
    public class CameraScript : MonoBehaviour
    {
        public GameObject faceDetect;
        public FaceDetectScript faceDetectScript;
        // Use this for initialization
        void Start()
        {
            faceDetectScript = faceDetect.GetComponent<FaceDetectScript>();
        }
        // Update is called once per frame
        void Update()
        {
            Vector2 facepos = faceDetectScript.facepos;
            Debug.Log("facepos:(" + facepos.x + " " + facepos.y + ")");
            float alpha = facepos.y * Mathf.PI;
            float beta = facepos.x * Mathf.PI;
            Vector3 pos;
            pos.x = -3.0f * Mathf.Sin(alpha) * Mathf.Cos(beta);
            pos.y = 3.0f * Mathf.Cos(alpha);
            pos.z = -3.0f * Mathf.Sin(alpha) * Mathf.Sin(beta);
            transform.position = pos;
            Vector3 relativePos = -transform.position;
            Quaternion rotation = Quaternion.LookRotation(relativePos);
            transform.rotation = rotation;
        }
    }
12. Copy the below code and paste it to "FaceDetectScript" file

using UnityEngine;

using System.Collections;

using System.Drawing;

using Emgu.CV;

using Emgu.Util;

using Emgu;

using Emgu.CV.Util;

using Emgu.CV.CvEnum;

using Emgu.CV.Structure;

using System.IO;

using System.Runtime.InteropServices;

public class FaceDetectScript : MonoBehaviour {

const int CAPTURE_WIDTH = 320;

    const int CAPTURE_HEIGHT = 240;

    public Vector2 facepos;

private CascadeClassifier cascade;

    private Capture capture;

public GameObject display;

const double Scale = 1.04;

    const double ScaleFactor = 1.139;

    const int MinNeighbors = 2;

// Use this for initialization

void Start ()

    {

        cascade = new CascadeClassifier("Assets/haarcascade_frontalface_alt.xml");

        

        capture = new Capture();

        capture.SetCaptureProperty(Emgu.CV.CvEnum.CapProp.FrameHeight, CAPTURE_HEIGHT);

        capture.SetCaptureProperty(Emgu.CV.CvEnum.CapProp.FrameWidth, CAPTURE_WIDTH);

        Mat currentframe = (capture.QueryFrame());

        Debug.Log("width:" + currentframe.Width + " height:" + currentframe.Height);

        

}

// Update is called once per frame

void Update () 

    {

Image<Bgr, byte> frame = capture.QueryFrame().ToImage<Bgr, byte>();

        //Display on top corner 

        MemoryStream _memory = new MemoryStream();

        frame.Bitmap.Save(_memory, frame.Bitmap.RawFormat);

        Texture2D camera = new Texture2D(400, 400);

        if (frame != null)

        {

            camera.LoadImage(_memory.ToArray());

            display.GetComponent<Renderer>().material.mainTexture = camera;

        }

        //Display on top corner

//moving the object

        Image<Gray, byte> gray = frame.Convert<Gray, byte>();

        using (Image<Bgr, byte> imageFrame = capture.QueryFrame().ToImage<Bgr, byte>())

        {

            if (imageFrame != null)

            {

                Image<Gray, byte> grayframe = imageFrame.Convert<Gray, byte>();

                Rectangle[] faces = cascade.DetectMultiScale(grayframe, 1.1, 10, Size.Empty, Size.Empty); //the actual face detection happens here

for (int i = 0; i < faces.Length; i++)

                {

                    CircleF _circle = new CircleF();

                    float x = (int)(faces[i].X) + (faces[i].Width / 2);

                    float y = (int)(faces[i].Y) + (faces[i].Height / 2);

                    _circle.Radius = faces[i].Width / 2;

                    imageFrame.Draw(new CircleF(new PointF(x,y),_circle.Radius), new Bgr(100,100,256));

                    Debug.Log("Circle Position tracking:(" + x + " " + y + ")");

                    if (faces.Length > 0)

                    {

                        facepos = new Vector2((x / 2.0f) / CAPTURE_WIDTH, (y / 2.0f) / CAPTURE_HEIGHT);

                    }

                }      

            }            

        }        

}

}

13. Now switch back to unity3D and select the Main Camera and Under the attached CameraScript drag and drop the FaceDetect game object on Face Detect field as shown in the below image.

14. Download and Copy the attached haarcascade_frontalface_alt.xml file and place it under the Assets Folder (link: https://github.com/gurunathanlakshmanan/EmguCV_Unity/blob/gh-pages/haarcascade_frontalface_alt.xml)

15. That's it now plug your webcam and run the game. 

16. Move your head so that you can see the Cube on your Game view too will rotate according.

17. That's it enjoy!