RunFaceLandmark.cs

using UnityEngine;
using Unity.Sentis;
using UnityEngine.Video;
using UnityEngine.UI;
using System.Collections.Generic;
using Lays = Unity.Sentis.Layers;

/*
 *                   Face Landmarks Inference
 *                   ========================
 *                   
 * Basic inference script for mediapose face landmarks
 * 
 * Put this script on the Main Camera
 * Put face_landmarks.sentis in the Assets/StreamingAssets folder
 * Create a RawImage of in the scene
 * Put a link to that image in previewUI
 * Put a video in Assets/StreamingAssets folder and put the name of it int videoName
 * Or put a test image in inputImage
 * Set inputType to appropriate input
 */


public class RunFaceLandmark : MonoBehaviour
{
    //Drag a link to a raw image here:
    public RawImage previewUI = null;

    public string videoName = "chatting.mp4";

    // Image to put into neural network
    public Texture2D inputImage;

    public InputType inputType = InputType.Video;

    //Resolution of displayed image
    Vector2Int resolution = new Vector2Int(640, 640);
    WebCamTexture webcam;
    VideoPlayer video;

    const BackendType backend = BackendType.GPUCompute;

    RenderTexture targetTexture;
    public enum InputType { Image, Video, Webcam };

    const int markerWidth = 5;

    //Holds array of colors to draw landmarks
    Color32[] markerPixels;

    IWorker worker;

    //Size of input image to neural network (196)
    const int size = 192;

    Ops ops;
    ITensorAllocator allocator;

    Model model;

    //webcam device name:
    const string deviceName = "";

    bool closing = false;

    Texture2D canvasTexture;

    void Start()
    {
        allocator = new TensorCachingAllocator();

        //(Note: if using a webcam on mobile get permissions here first)

        SetupTextures();
        SetupMarkers();
        SetupInput();
        SetupModel();
        SetupEngine();
    }

    void SetupModel()
    {
        model = ModelLoader.Load(Application.streamingAssetsPath + "/face_landmark.sentis");
    }
    public void SetupEngine()
    {
        worker = WorkerFactory.CreateWorker(backend, model);
        ops = WorkerFactory.CreateOps(backend, allocator);
    }

    void SetupTextures()
    {
        //To display the get and display the original image:
        targetTexture = new RenderTexture(resolution.x, resolution.y, 0);

        //Used for drawing the markers:
        canvasTexture = new Texture2D(targetTexture.width, targetTexture.height);

        previewUI.texture = targetTexture;
    }

    void SetupMarkers()
    {
        markerPixels = new Color32[markerWidth * markerWidth];
        for (int n = 0; n < markerWidth * markerWidth; n++)
        {
            markerPixels[n] = Color.white;
        }
        int center = markerWidth / 2;
        markerPixels[center * markerWidth + center] = Color.black;
    }

    void SetupInput()
    {
        switch (inputType)
        {
            case InputType.Webcam:
                {
                    webcam = new WebCamTexture(deviceName, resolution.x, resolution.y);
                    webcam.requestedFPS = 30;
                    webcam.Play();
                    break;
                }
            case InputType.Video:
                {
                    video = gameObject.AddComponent<VideoPlayer>();//new VideoPlayer();
                    video.renderMode = VideoRenderMode.APIOnly;
                    video.source = VideoSource.Url;
                    video.url = Application.streamingAssetsPath + "/"+videoName;
                    video.isLooping = true;
                    video.Play();
                    break;
                }
            default:
                {
                    Graphics.Blit(inputImage, targetTexture);
                }
                break;
        }
    }

    void Update()
    {
        GetImageFromSource();

        if (Input.GetKeyDown(KeyCode.Escape))
        {
            closing = true;
            Application.Quit();
        }

        if (Input.GetKeyDown(KeyCode.P))
        {
            previewUI.enabled = !previewUI.enabled;
        }
    }

    void GetImageFromSource()
    {
        if (inputType == InputType.Webcam)
        {
            // Format video input
            if (!webcam.didUpdateThisFrame) return;

            var aspect1 = (float)webcam.width / webcam.height;
            var aspect2 = (float)resolution.x / resolution.y;
            var gap = aspect2 / aspect1;

            var vflip = webcam.videoVerticallyMirrored;
            var scale = new Vector2(gap, vflip ? -1 : 1);
            var offset = new Vector2((1 - gap) / 2, vflip ? 1 : 0);

            Graphics.Blit(webcam, targetTexture, scale, offset);
        }
        if (inputType == InputType.Video)
        {
            var aspect1 = (float)video.width / video.height;
            var aspect2 = (float)resolution.x / resolution.y;
            var gap = aspect2 / aspect1;

            var vflip = false;
            var scale = new Vector2(gap, vflip ? -1 : 1);
            var offset = new Vector2((1 - gap) / 2, vflip ? 1 : 0);
            Graphics.Blit(video.texture, targetTexture, scale, offset);
        }
        if (inputType == InputType.Image)
        {
            Graphics.Blit(inputImage, targetTexture);
        }
    }

    void LateUpdate()
    {
        if (!closing)
        {
            RunInference(targetTexture);
        }
    }

    void RunInference(Texture source)
    {
        var transform = new TextureTransform();
        transform.SetDimensions(size, size, 3);
        transform.SetTensorLayout(0, 3, 1, 2);
        using var image0 = TextureConverter.ToTensor(source, transform);

        // Pre-process the image to make input in range (-1..1)
        using var image = ops.Mad(image0, 2f, -1f);

        worker.Execute(image0);

        using var landmarks= worker.PeekOutput("conv2d_21") as TensorFloat;
        
        //This gives the condifidence:
        //using var confidence = worker.PeekOutput("conv2d_31") as TensorFloat;

        float scaleX = targetTexture.width * 1f / size;
        float scaleY = targetTexture.height * 1f / size;

        landmarks.MakeReadable();
        DrawLandmarks(landmarks, scaleX, scaleY);
    }

    void DrawLandmarks(TensorFloat landmarks, float scaleX, float scaleY)
    {
        int numLandmarks = landmarks.shape[3] / 3; //468 face landmarks

        RenderTexture.active = targetTexture;
        canvasTexture.ReadPixels(new Rect(0, 0, targetTexture.width, targetTexture.height), 0, 0);

        for (int n = 0; n < numLandmarks; n++)
        {
            int px = (int)(landmarks[0, 0, 0, n * 3 + 0] * scaleX) - (markerWidth - 1) / 2;
            int py = (int)(landmarks[0, 0, 0, n * 3 + 1] * scaleY) - (markerWidth - 1) / 2;
            int pz = (int)(landmarks[0, 0, 0, n * 3 + 2] * scaleX);
            int destX = Mathf.Clamp(px, 0, targetTexture.width - 1 - markerWidth);
            int destY = Mathf.Clamp(targetTexture.height - 1 - py, 0, targetTexture.height - 1 - markerWidth);
            canvasTexture.SetPixels32(destX, destY, markerWidth, markerWidth, markerPixels);
        }
        canvasTexture.Apply();
        Graphics.Blit(canvasTexture, targetTexture);
        RenderTexture.active = null;
    }

    void CleanUp()
    {
        closing = true;
        ops?.Dispose();
        allocator?.Dispose();
        if (webcam) Destroy(webcam);
        if (video) Destroy(video);
        RenderTexture.active = null;
        targetTexture.Release();
        worker?.Dispose();
        worker = null;
    }

    void OnDestroy()
    {
        CleanUp();
    }

}