Xamarin NuGet Package >
Augmented Video for Android

Using the planar marker tutorial as a base, we are now going to add an augmented video to our app. The video will appear on top of a marker and show one of Pikkart's video ads.

First we need to copy some premade base classes from our sample package into the project dir. 

Copy the classes of the package into the root folder of the project (the folder where your MainActivity is). You will find three new c# classes in your app project as in the following image:

Change the namespace in the three classes to match the namespace of your app.

namespace PikkartSample.Droid
namespace Pikkart_SDK_tutorial

The FullscreenVideoPlayer class is a fullscreen video player activity, needed in case an android device doesn't support AR Videos. The PikkartVideoPlayer class is a class encapsulating an android MediaPlayer and an OpenGL rendering surface to witch video frames are redirected. This is the main class managing our video/audio data. The VideoMesh class is our 3D Object (a simple plane with our video as texture) that will be drawn on top of the tracked image. 

Make sure that the sample markers and media dirs from the sample package are copied in your app assets folder (<pikkart_sample_dir>/Assets/markers/ and <pikkart_sample_dir>/Assets/media/).

Once all is set-up, we need to change some bits of code in our ARRenderer class. First we need to add a new member variable for the VideoMesh object we need to draw, and we create and initialize it in the OnSurfaceCreated method:

public class ARRenderer : GLTextureView.Renderer {
...
private VideoMesh videoMesh = null;

public void OnSurfaceCreated(IGL10 gl, EGLConfig config)  {
    gl.GlClearColor(1.0f, 1.0f, 1.0f, 1.0f);
    monkeyMesh=new Mesh();

    Task.Run(async () =>
    {
        try
        {
            ((Activity)context).RunOnUiThread(() =>
            {
                progressDialog = ProgressDialog.Show(context, "Loading textures", "The 3D and video template textures of this tutorial have not been loaded yet", true);
            });

            monkeyMesh.InitMesh(context.Assets, "media/monkey.json", "media/texture.png");

            videoMesh.InitMesh(context.Assets, "media/pikkart_video.mp4", "media/pikkart_keyframe.png", 0, false, null);

            if (progressDialog != null)
                progressDialog.Dismiss();
        }
        catch (System.OperationCanceledException ex)
        {
            Console.WriteLine("init failed: {ex.Message}");
        }
        catch (Exception ex)
        {
            Console.WriteLine(ex.Message);
        }
    });
}

The VideoMesh is created and initialized by passing to it a reference to the current app AssetManager, the URL of the video file (can be a locally stored file or a web URL), the path to a keyframe image that will be shown to the user when the video is not playing (path to a locally stored image, in this case inside the media folder of the app assets folder) a starting seek position (in milliseconds), and whatever the video has to auto start of first detection. You can also pass a pointer to an external PikkartVideoPlayer object if you don't want the VideoMesh object to use its internal one.

Next we added a couple more support methods that modify and rotate the projection matrix depending on device screen orientation:

public bool ComputeModelViewProjectionMatrix(float[] mvMatrix, float[] pMatrix)
{
    RenderUtils.matrix44Identity(mvMatrix);
    RenderUtils.matrix44Identity(pMatrix);

    float w = (float)640;
    float h = (float)480;
    float ar = (float)ViewportHeight / (float)ViewportWidth;
    if (ViewportHeight > ViewportWidth) ar = 1.0f / ar;
    float h1 = h, w1 = w;
    if (ar < h / w)
        h1 = w * ar;
    else
        w1 = h / ar;

    float a = 0f, b = 0f;
    switch (Angle)
    {
        case 0:
            a = 1f; b = 0f;
            break;
        case 90:
            a = 0f; b = 1f;
            break;
        case 180:
            a = -1f; b = 0f;
            break;
        case 270:
            a = 0f; b = -1f;
            break;
        default: break;
    }
    float[] angleMatrix = new float[16];
    angleMatrix[0] = a; angleMatrix[1] = b; angleMatrix[2] = 0.0f; angleMatrix[3] = 0.0f;
    angleMatrix[4] = -b; angleMatrix[5] = a; angleMatrix[6] = 0.0f; angleMatrix[7] = 0.0f;
    angleMatrix[8] = 0.0f; angleMatrix[9] = 0.0f; angleMatrix[10] = 1.0f; angleMatrix[11] = 0.0f;
    angleMatrix[12] = 0.0f; angleMatrix[13] = 0.0f; angleMatrix[14] = 0.0f; angleMatrix[15] = 1.0f;

    float[] projectionMatrix = (float[])RecognitionFragment.GetCurrentProjectionMatrix().Clone();
    projectionMatrix[5] = projectionMatrix[5] * (h / h1);

    RenderUtils.matrixMultiply(4, 4, angleMatrix, 4, 4, projectionMatrix, pMatrix);

    if (RecognitionFragment.IsTracking)
    {
        float[] tMatrix = RecognitionFragment.GetCurrentModelViewMatrix();
        mvMatrix[0] = tMatrix[0]; mvMatrix[1] = tMatrix[1]; mvMatrix[2] = tMatrix[2]; mvMatrix[3] = tMatrix[3];
        mvMatrix[4] = tMatrix[4]; mvMatrix[5] = tMatrix[5]; mvMatrix[6] = tMatrix[6]; mvMatrix[7] = tMatrix[7];
        mvMatrix[8] = tMatrix[8]; mvMatrix[9] = tMatrix[9]; mvMatrix[10] = tMatrix[10]; mvMatrix[11] = tMatrix[11];
        mvMatrix[12] = tMatrix[12]; mvMatrix[13] = tMatrix[13]; mvMatrix[14] = tMatrix[14]; mvMatrix[15] = tMatrix[15];
        return true;
    }
    return false;
}

public bool ComputeProjectionMatrix(float[] pMatrix)
{
    RenderUtils.matrix44Identity(pMatrix);

    float w = (float)640;
    float h = (float)480;

    float ar = (float)ViewportHeight / (float)ViewportWidth;
    if (ViewportHeight > ViewportWidth) ar = 1.0f / ar;
    float h1 = h, w1 = w;
    if (ar < h / w)
        h1 = w * ar;
    else
        w1 = h / ar;

    float a = 0f, b = 0f;
    switch (Angle)
    {
        case 0:
            a = 1f; b = 0f;
            break;
        case 90:
            a = 0f; b = 1f;
            break;
        case 180:
            a = -1f; b = 0f;
            break;
        case 270:
            a = 0f; b = -1f;
            break;
        default: break;
    }

    float[] angleMatrix = new float[16];

    angleMatrix[0] = a; angleMatrix[1] = b; angleMatrix[2] = 0.0f; angleMatrix[3] = 0.0f;
    angleMatrix[4] = -b; angleMatrix[5] = a; angleMatrix[6] = 0.0f; angleMatrix[7] = 0.0f;
    angleMatrix[8] = 0.0f; angleMatrix[9] = 0.0f; angleMatrix[10] = 1.0f; angleMatrix[11] = 0.0f;
    angleMatrix[12] = 0.0f; angleMatrix[13] = 0.0f; angleMatrix[14] = 0.0f; angleMatrix[15] = 1.0f;

    float[] projectionMatrix = (float[])RecognitionFragment.GetCurrentProjectionMatrix().Clone();
    projectionMatrix[5] = projectionMatrix[5] * (h / h1);

    RenderUtils.matrixMultiply(4, 4, angleMatrix, 4, 4, projectionMatrix, pMatrix);

    return true;
}

We then proceed to modify the OnDrawFrame methods, we draw the video mesh if we recognize a specific marker, and also we make sure to still draw the video even if tracking of the marker is lost, this way:

/** Called to draw the current frame. */
public void OnDrawFrame(IGL10 gl)
{
    if (!IsActive) return;

    gl.GlClear(GL10.GlColorBufferBit | GL10.GlDepthBufferBit);

    // Call our native function to render camera content
    RecognitionFragment.RenderCamera(ViewportWidth, ViewportHeight, Angle);

    if (RecognitionFragment.IsTracking)
    {
        Marker currentMarker = RecognitionFragment.CurrentMarker;
        //Here we decide which 3d object to draw and we draw it
        if (currentMarker.Id.CompareTo("3_522") == 0)
        {
            float[] mvMatrix = new float[16];
            float[] pMatrix = new float[16];
            if (ComputeModelViewProjectionMatrix(mvMatrix, pMatrix))
            {
                if (videoMesh != null && videoMesh.MeshLoaded)
                {
                    if (videoMesh.GLLoaded)
                        videoMesh.DrawMesh(mvMatrix, pMatrix);
                    else
                        videoMesh.InitMeshGL();

                    RenderUtils.CheckGLError("completed video mesh Render");
                }
            }
        }
        else
        {
            float[] mvpMatrix = new float[16];
            if (ComputeModelViewProjectionMatrix(mvpMatrix))
            {
                //draw our 3d mesh on top of the marker
                if (monkeyMesh != null && monkeyMesh.MeshLoaded)
                {
                    if (monkeyMesh.GLLoaded)
                        monkeyMesh.DrawMesh(mvpMatrix);
                    else
                        monkeyMesh.InitMeshGL();

                    RenderUtils.CheckGLError("completed Monkey head Render");
                }
            }
        }
    }
    //if the video is still playing and we have lost tracking, we still draw the video, 
    //but in a fixed frontal position
    if (!RecognitionFragment.IsTracking && videoMesh != null && videoMesh.IsPlaying())
    {
        float[] mvMatrix = new float[16];
        float[] pMatrix = new float[16];
        ComputeProjectionMatrix(pMatrix);

        if (Angle == 0)
        {
            mvMatrix[0] = 1.0f; mvMatrix[1] = 0.0f; mvMatrix[2] = 0.0f; mvMatrix[3] = -0.5f;
            mvMatrix[4] = 0.0f; mvMatrix[5] = -1.0f; mvMatrix[6] = 0.0f; mvMatrix[7] = 0.4f;
            mvMatrix[8] = 0.0f; mvMatrix[9] = 0.0f; mvMatrix[10] = -1.0f; mvMatrix[11] = -1.3f;
            mvMatrix[12] = 0.0f; mvMatrix[13] = 0.0f; mvMatrix[14] = 0.0f; mvMatrix[15] = 1.0f;
        }
        else if (Angle == 90)
        {
            mvMatrix[0] = 0.0f; mvMatrix[1] = 1.0f; mvMatrix[2] = 0.0f; mvMatrix[3] = -0.5f;
            mvMatrix[4] = 1.0f; mvMatrix[5] = 0.0f; mvMatrix[6] = 0.0f; mvMatrix[7] = -0.5f;
            mvMatrix[8] = 0.0f; mvMatrix[9] = 0.0f; mvMatrix[10] = -1.0f; mvMatrix[11] = -1.3f;
            mvMatrix[12] = 0.0f; mvMatrix[13] = 0.0f; mvMatrix[14] = 0.0f; mvMatrix[15] = 1.0f;
        }
        else if (Angle == 180)
        {
            mvMatrix[0] = -1.0f; mvMatrix[1] = 0.0f; mvMatrix[2] = 0.0f; mvMatrix[3] = 0.5f;
            mvMatrix[4] = 0.0f; mvMatrix[5] = 1.0f; mvMatrix[6] = 0.0f; mvMatrix[7] = -0.4f;
            mvMatrix[8] = 0.0f; mvMatrix[9] = 0.0f; mvMatrix[10] = -1.0f; mvMatrix[11] = -1.3f;
            mvMatrix[12] = 0.0f; mvMatrix[13] = 0.0f; mvMatrix[14] = 0.0f; mvMatrix[15] = 1.0f;
        }
        else if (Angle == 270)
        {
            mvMatrix[0] = 0.0f; mvMatrix[1] = -1.0f; mvMatrix[2] = 0.0f; mvMatrix[3] = 0.5f;
            mvMatrix[4] = -1.0f; mvMatrix[5] = 0.0f; mvMatrix[6] = 0.0f; mvMatrix[7] = 0.5f;
            mvMatrix[8] = 0.0f; mvMatrix[9] = 0.0f; mvMatrix[10] = -1.0f; mvMatrix[11] = -1.3f;
            mvMatrix[12] = 0.0f; mvMatrix[13] = 0.0f; mvMatrix[14] = 0.0f; mvMatrix[15] = 1.0f;
        }

        videoMesh.DrawMesh(mvMatrix, pMatrix);
        RenderUtils.CheckGLError("completed video mesh Render");
    }
    gl.GlFinish();
}

Lastly for our ARRenderer class, we add two support functions that will either play or pause the video. These will be used by our MainActivity to respond to user interaction and app state change (OnPause and OnResume mainly)

public void PlayOrPauseVideo() {
    if (videoMesh != null) videoMesh.PlayOrPauseVideo();
}

public void PauseVideo() { if (videoMesh != null) videoMesh.PauseVideo(); }

It's time to modify the ARView class. It just requires the addition of a couple of functions: one function to respond to user input (tap on screen) and a second function to pause video playing:

public override bool OnTouchEvent(MotionEvent e)
{
    if (e.Action == MotionEventActions.Up)
        _renderer.PlayOrPauseVideo();
    return true;
}

Lastly we need to override two member functions of our MainActivity class in order to correctly handle the app state. First make a class member variable m_arView to hold a reference to our ARView view that was created in the main activity InitLayout function, then we need to override the OnPause and OnResume function in this way to correctly pause the videos:

public class MainActivity : AppCompatActivity, IRecognitionListener
    ...
    private ARView m_arView = null;
    ...
    private void InitLayout()
    {
        SetContentView(Resource.Layout.Main);

        m_arView = new ARView(this);
        AddContentView(m_arView, new FrameLayout.LayoutParams(FrameLayout.LayoutParams.MatchParent, FrameLayout.LayoutParams.MatchParent));

        _cameraFragment = FragmentManager.FindFragmentById<RecognitionFragment>(Resource.Id.ar_fragment);
        _cameraFragment.StartRecognition(new RecognitionOptions(RecognitionOptions.RecognitionStorage.Local, RecognitionOptions.RecognitionMode.ContinuousScan,
            new CloudRecognitionInfo(new String[] { })), this);
    }
    ...

   protected override void OnResume()
        base.OnResume();
        //restart recognition on app resume
        _cameraFragment = FragmentManager.FindFragmentById<RecognitionFragment>(Resource.Id.ar_fragment);
        if (_cameraFragment != null) _cameraFragment.StartRecognition(
                new RecognitionOptions(
                    RecognitionOptions.RecognitionStorage.Local,
                    RecognitionOptions.RecognitionMode.ContinuousScan,
                    new CloudRecognitionInfo(new String[] { })
                ), this);
        //resume our renderer
        if (m_arView != null) m_arView.onResume();
    }

    protected override void OnPause()
        base.OnPause();
        //pause our renderer and associated videos
        if (m_arView != null) m_arView.onPause();
    }

And that's all folks for this tutorial. Once compiled and run on a device you should see something like this when targeting one of our test markers:

If you want more details on how our VideoMesh is made and how our AR video playback works take a look at the code and read the comments of our VideoMesh and PikkartVideoPlayer classes.

You can download a complete project for augmented video from Github @ https://github.com/pikkart-support/XamarinAndroid_AugmentedVideo (remember to add you license file in the assets folder!).