DeepAR.h

Go to the documentation of this file.

 
#import <Foundation/Foundation.h>
#import <UIKit/UIKit.h>
#import <AVFoundation/AVFoundation.h>
 
typedef struct {
    BOOL detected;
    float translation[3];
    float rotation[3];
    float poseMatrix[16];
    float landmarks[68 * 3];
    float landmarks2d[68 * 3]; // TODO: *3 is definitely a typo; fix and test
    float faceRect[4];
    float emotions[5];
    float actionUnits[63]; // TODO: discover more
    int numberOfActionUnits; // TODO: discover more
} FaceData;
 
typedef struct {
    FaceData faceData[4];
} MultiFaceData;
 
typedef struct {
    NSInteger outputWidth;
    NSInteger outputHeight;
    CGRect subframe;
    NSDictionary* videoCompressionProperties;
    BOOL recordAudio;
} RecordingConfig;
 
typedef struct {
    float x;
    float y;
    float z;
    float w;
} Vector4;
 
typedef struct {    
    float x;
    float y;
    float z;
} Vector3;
 
typedef enum
{
    Undefined, // 0
    RGBA,      // 1
    BGRA,      // 2
    ARGB,      // 3
    ABGR,      // 4
    COUNT
} OutputFormat;
 
typedef enum {
    DEEPAR_ERROR_TYPE_DEBUG,
    DEEPAR_ERROR_TYPE_INFO,
    DEEPAR_ERROR_TYPE_WARNING,
    DEEPAR_ERROR_TYPE_ERROR
} ARErrorType;
 
typedef enum {
    START,
    MOVE,
    END
} TouchType;
 
typedef enum {
    BOOLEAN = 0,
    INT = 1,
    DOUBLE = 2,
    STRING = 3
} VarType;
 
typedef struct {
    CGFloat x;
    CGFloat y;
    TouchType type;
} TouchInfo;
 
typedef struct {
    bool initializeEngineWithFaceTracking;
 
    bool initializeFaceTrackingSynchronously;
} FaceTrackingInitParameters;
 
@protocol DeepARDelegate <NSObject>
 
@optional
 
- (void)didTakeScreenshot:(UIImage*)screenshot;
 
- (void)didInitialize;
 
- (void)faceVisiblityDidChange:(BOOL)faceVisible;
 
- (void)frameAvailable:(CMSampleBufferRef)sampleBuffer;
 
- (void)faceTracked:(MultiFaceData)faceData;
 
- (void)numberOfFacesVisibleChanged:(NSInteger)facesVisible;
 
- (void)didFinishShutdown;
 
- (void)imageVisibilityChanged:(NSString*)gameObjectName imageVisible:(BOOL)imageVisible; // TODO: explore further
 
- (void)didSwitchEffect:(NSString*)slot;
 
- (void)animationTransitionedToState:(NSString*)state;
 
- (void)didStartVideoRecording;
 
- (void)didFinishPreparingForVideoRecording;
 
- (void)didFinishVideoRecording:(NSString*)videoFilePath;
 
- (void)recordingFailedWithError:(NSError*)error;
 
- (void)onErrorWithCode:(ARErrorType)code error:(NSString*)error;
 
@end
 
#define ARViewDelegate DeepARDelegate
 
@interface DeepAR : NSObject
 
@property (nonatomic, weak) id<DeepARDelegate> delegate;
 
@property (nonatomic, readonly) BOOL visionInitialized;
 
@property (nonatomic, readonly) BOOL renderingInitialized;
 
@property (nonatomic, readonly) BOOL faceVisible;
 
@property (nonatomic, readonly) CGSize renderingResolution;
 
@property (nonatomic, assign) BOOL videoRecordingWarmupEnabled;
 
@property (nonatomic, strong) NSDictionary* audioCompressionSettings;
 
+ (NSString*) sdkVersion;
 
- (void)setLicenseKey:(NSString*)key;
 
- (void)initializeWithWidth:(NSInteger)width height:(NSInteger)height window:(CAEAGLLayer*)window;
 
- (void)initialize;
 
- (void)initializeOffscreenWithWidth:(NSInteger)width height:(NSInteger)height;
 
- (UIView*)createARViewWithFrame:(CGRect)frame;
 
- (BOOL)isVisionOnly;
 
- (void)setRenderingResolutionWithWidth:(NSInteger)width height:(NSInteger)height;
 
- (void)shutdown;
 
- (UIView*)switchToRenderingToViewWithFrame:(CGRect)frame;
 
- (void)switchToRenderingOffscreenWithWidth:(NSInteger)width height:(NSInteger)height;
 
- (void)changeLiveMode:(BOOL)liveMode;
 
- (void)resume;
 
- (void)pause;
 
- (void)processFrame:(CVPixelBufferRef)imageBuffer mirror:(BOOL)mirror;
 
- (void)processFrame:(CVPixelBufferRef)imageBuffer mirror:(BOOL)mirror timestamp:(CMTimeValue)timestamp;
 
- (void)processFrameAndReturn:(CVPixelBufferRef)imageBuffer outputBuffer:(CVPixelBufferRef)outputBuffer mirror:(BOOL)mirror;
 
- (void)enqueueCameraFrame:(CMSampleBufferRef)sampleBuffer mirror:(BOOL)mirror;
 
- (void)enqueueAudioSample:(CMSampleBufferRef)sampleBuffer;
 
- (void)setVideoRecordingOutputName:(NSString*)outputName;
 
- (void)setVideoRecordingOutputPath:(NSString*)outputPath;
 
- (void)startVideoRecordingWithOutputWidth:(int)outputWidth outputHeight:(int)outputHeight;
 
- (void)startVideoRecordingWithOutputWidth:(int)outputWidth outputHeight:(int)outputHeight subframe:(CGRect)subframe;
 
- (void)startVideoRecordingWithOutputWidth:(int)outputWidth outputHeight:(int)outputHeight subframe:(CGRect)subframe videoCompressionProperties:(NSDictionary*)videoCompressionProperties;
 
- (void)startVideoRecordingWithOutputWidth:(int)outputWidth outputHeight:(int)outputHeight subframe:(CGRect)subframe videoCompressionProperties:(NSDictionary*)videoCompressionProperties recordAudio:(BOOL)recordAudio;
 
- (void)switchEffectWithSlot:(NSString*)slot path:(NSString*)path;
 
- (void)switchEffectWithSlot:(NSString*)slot path:(NSString*)path face:(NSInteger)face;
 
- (void)switchEffectWithSlot:(NSString*)slot path:(NSString*)path face:(NSInteger)face targetGameObject:(NSString*)targetGameObject;
 
- (void)switchEffectWithSlot:(NSString*)slot data:(NSData*)data;
 
- (void)switchEffectWithSlot:(NSString*)slot data:(NSData*)data face:(NSInteger)face;
 
- (void)takeScreenshot;
 
- (void)finishVideoRecording;
 
- (void)pauseVideoRecording;
 
- (void)resumeVideoRecording;
 
- (void)enableAudioProcessing:(BOOL)enabled;
 
- (void)setAudioProcessingSemitone:(float)sts;
 
- (void)startCaptureWithOutputWidth:(NSInteger)outputWidth outputHeight:(NSInteger)outputHeight subframe:(CGRect)subframe;
 
- (void)startCaptureWithOutputWidthAndFormat:(NSInteger)outputWidth outputHeight:(NSInteger)outputHeight subframe:(CGRect)subframe outputImageFormat:(OutputFormat)outputFormat;
 
- (void)stopCapture;
 
- (void)fireTrigger:(NSString*)trigger;
 
- (void)touchOccurred:(TouchInfo)touchInfo;
 
- (void)showStats:(BOOL) enabled;
 
- (void)simulatePhysics:(BOOL) enabled;
 
- (void)showColliders:(BOOL) enabled;
 
- (void)setFaceDetectionSensitivity:(NSInteger)sensitivity;
 
- (void)changeParameter:(NSString*)gameObject component:(NSString*)component parameter:(NSString*)parameter floatValue:(float)value;
 
- (void)changeParameter:(NSString*)gameObject component:(NSString*)component parameter:(NSString*)parameter vectorValue:(Vector4)value;
 
- (void)changeParameter:(NSString*)gameObject component:(NSString*)component parameter:(NSString*)parameter vector3Value:(Vector3)value;
 
- (void)changeParameter:(NSString*)gameObject component:(NSString*)component parameter:(NSString*)parameter boolValue:(BOOL)value;
 
- (void)changeParameter:(NSString*)gameObject component:(NSString*)component parameter:(NSString*)parameter image:(UIImage*)image;
 
- (void)changeParameter:(NSString*)gameObject component:(NSString*)component parameter:(NSString*)parameter stringValue:(NSString*)value;
 
 
- (void)moveGameObject:(NSString*)selectedGameObjectName targetGameObjectname:(NSString*)targetGameObjectName;
 
- (void)startProfiling;
 
- (void)stopProfiling;
 
- (void)setFaceTrackingInitParameters:(FaceTrackingInitParameters)params;
 
- (bool)hasVar:(NSString*)name slot:(NSString*) slot;
 
- (bool)hasVar:(NSString*)name;
 
- (VarType)getVarType:(NSString*)name slot:(NSString*) slot;
 
- (VarType)getVarType:(NSString*)name;
 
- (bool)getBoolVar:(NSString*)name slot:(NSString*) slot;
 
- (bool)getBoolVar:(NSString*)name;
 
- (int)getIntVar:(NSString*)name slot:(NSString*) slot;
 
- (int)getIntVar:(NSString*)name;
 
- (double)getDoubleVar:(NSString*)name slot:(NSString*) slot;
 
- (double)getDoubleVar:(NSString*)name;
 
- (NSString*)getStringVar:(NSString*)name slot:(NSString*) slot;
 
- (NSString*)getStringVar:(NSString*)name;
 
- (bool)setBoolVar:(NSString*)name value:(bool)value slot:(NSString*) slot;
 
- (bool)setBoolVar:(NSString*)name value:(bool)value;
 
- (bool)setIntVar:(NSString*)name value:(int)value slot:(NSString*) slot;
 
- (bool)setIntVar:(NSString*)name value:(int)value;
 
- (bool)setDoubleVar:(NSString*)name value:(double)value slot:(NSString*) slot;
 
- (bool)setDoubleVar:(NSString*)name value:(double)value;
 
- (bool)setStringVar:(NSString*)name value:(NSString*)value slot:(NSString*) slot;
 
- (bool)setStringVar:(NSString*)name value:(NSString*)value;
 
- (bool)deleteVar:(NSString*)name slot:(NSString*) slot;
 
- (bool)deleteVar:(NSString*)name;
 
- (bool)clearVars:(NSString*)slot;
 
- (bool)clearVars;
 
@end
 
#import "ARView.h"
#import "CameraController.h"