init
This commit is contained in:
441
Assets/Oculus/LipSync/Scripts/OVRLipSync.cs
Normal file
441
Assets/Oculus/LipSync/Scripts/OVRLipSync.cs
Normal file
@@ -0,0 +1,441 @@
|
||||
/************************************************************************************
|
||||
Filename : OVRLipSync.cs
|
||||
Content : Interface to Oculus Lip Sync engine
|
||||
Created : August 4th, 2015
|
||||
Copyright : Copyright Facebook Technologies, LLC and its affiliates.
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Oculus Audio SDK License Version 3.3 (the "License");
|
||||
you may not use the Oculus Audio SDK except in compliance with the License,
|
||||
which is provided at the time of installation or download, or which
|
||||
otherwise accompanies this software in either electronic or hard copy form.
|
||||
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://developer.oculus.com/licenses/audio-3.3/
|
||||
|
||||
Unless required by applicable law or agreed to in writing, the Oculus Audio SDK
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
************************************************************************************/
|
||||
using UnityEngine;
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
//-------------------------------------------------------------------------------------
|
||||
// ***** OVRLipSync
|
||||
//
|
||||
/// <summary>
|
||||
/// OVRLipSync interfaces into the Oculus lip sync engine. This component should be added
|
||||
/// into the scene once.
|
||||
///
|
||||
/// </summary>
|
||||
public class OVRLipSync : MonoBehaviour
|
||||
{
|
||||
// Error codes that may return from Lip Sync engine
|
||||
public enum Result
|
||||
{
|
||||
Success = 0,
|
||||
Unknown = -2200, //< An unknown error has occurred
|
||||
CannotCreateContext = -2201, //< Unable to create a context
|
||||
InvalidParam = -2202, //< An invalid parameter, e.g. NULL pointer or out of range
|
||||
BadSampleRate = -2203, //< An unsupported sample rate was declared
|
||||
MissingDLL = -2204, //< The DLL or shared library could not be found
|
||||
BadVersion = -2205, //< Mismatched versions between header and libs
|
||||
UndefinedFunction = -2206 //< An undefined function
|
||||
};
|
||||
|
||||
// Audio buffer data type
|
||||
public enum AudioDataType
|
||||
{
|
||||
// Signed 16-bit integer mono audio stream
|
||||
S16_Mono,
|
||||
// Signed 16-bit integer stereo audio stream
|
||||
S16_Stereo,
|
||||
// Signed 32-bit float mono audio stream
|
||||
F32_Mono,
|
||||
// Signed 32-bit float stereo audio stream
|
||||
F32_Stereo
|
||||
};
|
||||
|
||||
// Various visemes
|
||||
public enum Viseme
|
||||
{
|
||||
sil,
|
||||
PP,
|
||||
FF,
|
||||
TH,
|
||||
DD,
|
||||
kk,
|
||||
CH,
|
||||
SS,
|
||||
nn,
|
||||
RR,
|
||||
aa,
|
||||
E,
|
||||
ih,
|
||||
oh,
|
||||
ou
|
||||
};
|
||||
|
||||
public static readonly int VisemeCount = Enum.GetNames(typeof(Viseme)).Length;
|
||||
|
||||
// Enum for sending lip-sync engine specific signals
|
||||
public enum Signals
|
||||
{
|
||||
VisemeOn,
|
||||
VisemeOff,
|
||||
VisemeAmount,
|
||||
VisemeSmoothing,
|
||||
LaughterAmount
|
||||
};
|
||||
|
||||
public static readonly int SignalCount = Enum.GetNames(typeof(Signals)).Length;
|
||||
|
||||
// Enum for provider context to create
|
||||
public enum ContextProviders
|
||||
{
|
||||
Original,
|
||||
Enhanced,
|
||||
Enhanced_with_Laughter,
|
||||
};
|
||||
|
||||
/// NOTE: Opaque typedef for lip-sync context is an unsigned int (uint)
|
||||
|
||||
/// Current phoneme frame results
|
||||
[System.Serializable]
|
||||
public class Frame
|
||||
{
|
||||
public void CopyInput(Frame input)
|
||||
{
|
||||
frameNumber = input.frameNumber;
|
||||
frameDelay = input.frameDelay;
|
||||
input.Visemes.CopyTo(Visemes, 0);
|
||||
laughterScore = input.laughterScore;
|
||||
}
|
||||
|
||||
public void Reset()
|
||||
{
|
||||
frameNumber = 0;
|
||||
frameDelay = 0;
|
||||
Array.Clear(Visemes, 0, VisemeCount);
|
||||
laughterScore = 0;
|
||||
}
|
||||
|
||||
public int frameNumber; // count from start of recognition
|
||||
public int frameDelay; // in ms
|
||||
public float[] Visemes = new float[VisemeCount]; // Array of floats for viseme frame. Size of Viseme Count, above
|
||||
public float laughterScore; // probability of laughter presence.
|
||||
};
|
||||
|
||||
// * * * * * * * * * * * * *
|
||||
// Import functions
|
||||
#if !UNITY_IOS || UNITY_EDITOR
|
||||
public const string strOVRLS = "OVRLipSync";
|
||||
#else
|
||||
public const string strOVRLS = "__Internal";
|
||||
#endif
|
||||
[DllImport(strOVRLS)]
|
||||
private static extern int ovrLipSyncDll_Initialize(int samplerate, int buffersize);
|
||||
[DllImport(strOVRLS)]
|
||||
private static extern void ovrLipSyncDll_Shutdown();
|
||||
[DllImport(strOVRLS)]
|
||||
private static extern IntPtr ovrLipSyncDll_GetVersion(ref int Major,
|
||||
ref int Minor,
|
||||
ref int Patch);
|
||||
[DllImport(strOVRLS)]
|
||||
private static extern int ovrLipSyncDll_CreateContextEx(ref uint context,
|
||||
ContextProviders provider,
|
||||
int sampleRate,
|
||||
bool enableAcceleration);
|
||||
|
||||
[DllImport(strOVRLS)]
|
||||
private static extern int ovrLipSyncDll_CreateContextWithModelFile(ref uint context,
|
||||
ContextProviders provider,
|
||||
string modelPath,
|
||||
int sampleRate,
|
||||
bool enableAcceleration);
|
||||
|
||||
[DllImport(strOVRLS)]
|
||||
private static extern int ovrLipSyncDll_DestroyContext(uint context);
|
||||
|
||||
|
||||
[DllImport(strOVRLS)]
|
||||
private static extern int ovrLipSyncDll_ResetContext(uint context);
|
||||
[DllImport(strOVRLS)]
|
||||
private static extern int ovrLipSyncDll_SendSignal(uint context,
|
||||
Signals signal,
|
||||
int arg1, int arg2);
|
||||
[DllImport(strOVRLS)]
|
||||
private static extern int ovrLipSyncDll_ProcessFrameEx(
|
||||
uint context,
|
||||
IntPtr audioBuffer,
|
||||
uint bufferSize,
|
||||
AudioDataType dataType,
|
||||
ref int frameNumber,
|
||||
ref int frameDelay,
|
||||
float[] visemes,
|
||||
int visemeCount,
|
||||
ref float laughterScore,
|
||||
float[] laughterCategories,
|
||||
int laughterCategoriesLength);
|
||||
|
||||
// * * * * * * * * * * * * *
|
||||
// Public members
|
||||
|
||||
// * * * * * * * * * * * * *
|
||||
// Static members
|
||||
private static Result sInitialized = Result.Unknown;
|
||||
|
||||
// interface through this static member.
|
||||
public static OVRLipSync sInstance = null;
|
||||
|
||||
|
||||
// * * * * * * * * * * * * *
|
||||
// MonoBehaviour overrides
|
||||
|
||||
/// <summary>
|
||||
/// Awake this instance.
|
||||
/// </summary>
|
||||
void Awake()
|
||||
{
|
||||
// We can only have one instance of OVRLipSync in a scene (use this for local property query)
|
||||
if (sInstance == null)
|
||||
{
|
||||
sInstance = this;
|
||||
}
|
||||
else
|
||||
{
|
||||
Debug.LogWarning(System.String.Format("OVRLipSync Awake: Only one instance of OVRPLipSync can exist in the scene."));
|
||||
return;
|
||||
}
|
||||
|
||||
if (IsInitialized() != Result.Success)
|
||||
{
|
||||
sInitialized = Initialize();
|
||||
|
||||
if (sInitialized != Result.Success)
|
||||
{
|
||||
Debug.LogWarning(System.String.Format
|
||||
("OvrLipSync Awake: Failed to init Speech Rec library"));
|
||||
}
|
||||
}
|
||||
|
||||
// Important: Use the touchpad mechanism for input, call Create on the OVRTouchpad helper class
|
||||
OVRTouchpad.Create();
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Raises the destroy event.
|
||||
/// </summary>
|
||||
void OnDestroy()
|
||||
{
|
||||
if (sInstance != this)
|
||||
{
|
||||
Debug.LogWarning(
|
||||
"OVRLipSync OnDestroy: This is not the correct OVRLipSync instance.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Do not shut down at this time
|
||||
// ovrLipSyncDll_Shutdown();
|
||||
// sInitialized = (int)Result.Unknown;
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * *
|
||||
// Public Functions
|
||||
|
||||
public static Result Initialize()
|
||||
{
|
||||
int sampleRate;
|
||||
int bufferSize;
|
||||
int numbuf;
|
||||
|
||||
// Get the current sample rate
|
||||
sampleRate = AudioSettings.outputSampleRate;
|
||||
// Get the current buffer size and number of buffers
|
||||
AudioSettings.GetDSPBufferSize(out bufferSize, out numbuf);
|
||||
|
||||
String str = System.String.Format
|
||||
("OvrLipSync Awake: Queried SampleRate: {0:F0} BufferSize: {1:F0}", sampleRate, bufferSize);
|
||||
Debug.LogWarning(str);
|
||||
|
||||
sInitialized = (Result)ovrLipSyncDll_Initialize(sampleRate, bufferSize);
|
||||
return sInitialized;
|
||||
}
|
||||
|
||||
public static Result Initialize(int sampleRate, int bufferSize)
|
||||
{
|
||||
String str = System.String.Format
|
||||
("OvrLipSync Awake: Queried SampleRate: {0:F0} BufferSize: {1:F0}", sampleRate, bufferSize);
|
||||
Debug.LogWarning(str);
|
||||
|
||||
sInitialized = (Result)ovrLipSyncDll_Initialize(sampleRate, bufferSize);
|
||||
return sInitialized;
|
||||
}
|
||||
|
||||
public static void Shutdown()
|
||||
{
|
||||
ovrLipSyncDll_Shutdown();
|
||||
sInitialized = Result.Unknown;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines if is initialized.
|
||||
/// </summary>
|
||||
/// <returns><c>true</c> if is initialized; otherwise, <c>false</c>.</returns>
|
||||
public static Result IsInitialized()
|
||||
{
|
||||
return sInitialized;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a lip-sync context.
|
||||
/// </summary>
|
||||
/// <returns>error code</returns>
|
||||
/// <param name="context">Context.</param>
|
||||
/// <param name="provider">Provider.</param>
|
||||
/// <param name="enableAcceleration">Enable DSP Acceleration.</param>
|
||||
public static Result CreateContext(
|
||||
ref uint context,
|
||||
ContextProviders provider,
|
||||
int sampleRate = 0,
|
||||
bool enableAcceleration = false)
|
||||
{
|
||||
if (IsInitialized() != Result.Success && Initialize() != Result.Success)
|
||||
return Result.CannotCreateContext;
|
||||
|
||||
return (Result)ovrLipSyncDll_CreateContextEx(ref context, provider, sampleRate, enableAcceleration);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a lip-sync context with specified model file.
|
||||
/// </summary>
|
||||
/// <returns>error code</returns>
|
||||
/// <param name="context">Context.</param>
|
||||
/// <param name="provider">Provider.</param>
|
||||
/// <param name="modelPath">Model Dir.</param>
|
||||
/// <param name="sampleRate">Sampling Rate.</param>
|
||||
/// <param name="enableAcceleration">Enable DSP Acceleration.</param>
|
||||
public static Result CreateContextWithModelFile(
|
||||
ref uint context,
|
||||
ContextProviders provider,
|
||||
string modelPath,
|
||||
int sampleRate = 0,
|
||||
bool enableAcceleration = false)
|
||||
{
|
||||
if (IsInitialized() != Result.Success && Initialize() != Result.Success)
|
||||
return Result.CannotCreateContext;
|
||||
|
||||
return (Result)ovrLipSyncDll_CreateContextWithModelFile(
|
||||
ref context,
|
||||
provider,
|
||||
modelPath,
|
||||
sampleRate,
|
||||
enableAcceleration);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Destroy a lip-sync context.
|
||||
/// </summary>
|
||||
/// <returns>The context.</returns>
|
||||
/// <param name="context">Context.</param>
|
||||
public static Result DestroyContext(uint context)
|
||||
{
|
||||
if (IsInitialized() != Result.Success)
|
||||
return Result.Unknown;
|
||||
|
||||
return (Result)ovrLipSyncDll_DestroyContext(context);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resets the context.
|
||||
/// </summary>
|
||||
/// <returns>error code</returns>
|
||||
/// <param name="context">Context.</param>
|
||||
public static Result ResetContext(uint context)
|
||||
{
|
||||
if (IsInitialized() != Result.Success)
|
||||
return Result.Unknown;
|
||||
|
||||
return (Result)ovrLipSyncDll_ResetContext(context);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends a signal to the lip-sync engine.
|
||||
/// </summary>
|
||||
/// <returns>error code</returns>
|
||||
/// <param name="context">Context.</param>
|
||||
/// <param name="signal">Signal.</param>
|
||||
/// <param name="arg1">Arg1.</param>
|
||||
/// <param name="arg2">Arg2.</param>
|
||||
public static Result SendSignal(uint context, Signals signal, int arg1, int arg2)
|
||||
{
|
||||
if (IsInitialized() != Result.Success)
|
||||
return Result.Unknown;
|
||||
|
||||
return (Result)ovrLipSyncDll_SendSignal(context, signal, arg1, arg2);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process float[] audio buffer by lip-sync engine.
|
||||
/// </summary>
|
||||
/// <returns>error code</returns>
|
||||
/// <param name="context">Context.</param>
|
||||
/// <param name="audioBuffer"> PCM audio buffer.</param>
|
||||
/// <param name="frame">Lip-sync Frame.</param>
|
||||
/// <param name="stereo">Whether buffer is part of stereo or mono stream.</param>
|
||||
public static Result ProcessFrame(
|
||||
uint context, float[] audioBuffer, Frame frame, bool stereo = true)
|
||||
{
|
||||
if (IsInitialized() != Result.Success)
|
||||
return Result.Unknown;
|
||||
|
||||
var dataType = stereo ? AudioDataType.F32_Stereo : AudioDataType.F32_Mono;
|
||||
var numSamples = (uint)(stereo ? audioBuffer.Length / 2 : audioBuffer.Length);
|
||||
var handle = GCHandle.Alloc(audioBuffer, GCHandleType.Pinned);
|
||||
var rc = ovrLipSyncDll_ProcessFrameEx(context,
|
||||
handle.AddrOfPinnedObject(), numSamples, dataType,
|
||||
ref frame.frameNumber, ref frame.frameDelay,
|
||||
frame.Visemes, frame.Visemes.Length,
|
||||
ref frame.laughterScore,
|
||||
null, 0
|
||||
);
|
||||
handle.Free();
|
||||
return (Result)rc;
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process short[] audio buffer by lip-sync engine.
|
||||
/// </summary>
|
||||
/// <returns>error code</returns>
|
||||
/// <param name="context">Context.</param>
|
||||
/// <param name="audioBuffer"> PCM audio buffer.</param>
|
||||
/// <param name="frame">Lip-sync Frame.</param>
|
||||
/// <param name="stereo">Whether buffer is part of stereo or mono stream.</param>
|
||||
public static Result ProcessFrame(
|
||||
uint context, short[] audioBuffer, Frame frame, bool stereo = true)
|
||||
{
|
||||
if (IsInitialized() != Result.Success)
|
||||
return Result.Unknown;
|
||||
|
||||
var dataType = stereo ? AudioDataType.S16_Stereo : AudioDataType.S16_Mono;
|
||||
var numSamples = (uint)(stereo ? audioBuffer.Length / 2 : audioBuffer.Length);
|
||||
var handle = GCHandle.Alloc(audioBuffer, GCHandleType.Pinned);
|
||||
var rc = ovrLipSyncDll_ProcessFrameEx(context,
|
||||
handle.AddrOfPinnedObject(), numSamples, dataType,
|
||||
ref frame.frameNumber, ref frame.frameDelay,
|
||||
frame.Visemes, frame.Visemes.Length,
|
||||
ref frame.laughterScore,
|
||||
null, 0
|
||||
);
|
||||
handle.Free();
|
||||
return (Result)rc;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user