Skip to content
Snippets Groups Projects
Commit b72361bf authored by Victor's avatar Victor
Browse files

Update AprilTagDetector and add DebugLog utility

Added:
- Public property `isActivated` in `AprilTagDetector`
- Public method `AddTag`, `RemoveTag`, and `SetTagSizes` in `AprilTagDetector`
- Tooltip for `AprilTagDetector` serialized fields
- `DebugLog` utility based on custom scripting symbols
parent d5f414e9
No related branches found
No related tags found
No related merge requests found
Showing with 732 additions and 574 deletions
...@@ -7,7 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ...@@ -7,7 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
## 1.0.1 - 2025-06-24
### Added
- Public property `isActivated` in `AprilTagDetector`
- Public method `AddTag`, `RemoveTag`, and `SetTagSizes` in `AprilTagDetector`
- Tooltip for `AprilTagDetector` serialized fields
- `DebugLog` utility based on custom scripting symbols
## 1.0.0 - 2025-06-17 ## 1.0.0 - 2025-06-17
......
...@@ -23,12 +23,18 @@ This package is available in the `HYPER` scoped registry. ...@@ -23,12 +23,18 @@ This package is available in the `HYPER` scoped registry.
Please follow [this snippet] to add the registry to your project. Please follow [this snippet] to add the registry to your project.
## How to use
### Enable debug messages
Add `HYPER_PCA_APRILTAG_DEBUG_LOG` to **Project Settings** > **Player** > **Script Compilation** > **Scripting Define Symbols**.
## Attribution
- Correct Blips by CogFireStudios -- https://freesound.org/s/531510/ -- License: Creative Commons 0
[jp.keijiro.apriltag]: https://github.com/keijiro/jp.keijiro.apriltag [jp.keijiro.apriltag]: https://github.com/keijiro/jp.keijiro.apriltag
[LICENSE.BSD-2-Clause]: https://git-st.inf.tu-dresden.de/hyper/pca-apriltag/-/blob/main/LICENSE.BSD-2-Clause [LICENSE.BSD-2-Clause]: https://git-st.inf.tu-dresden.de/hyper/pca-apriltag/-/blob/main/LICENSE.BSD-2-Clause
[Unity-PassthroughCameraApiSamples]: https://github.com/oculus-samples/Unity-PassthroughCameraApiSamples/tree/main/Assets/PassthroughCameraApiSamples [Unity-PassthroughCameraApiSamples]: https://github.com/oculus-samples/Unity-PassthroughCameraApiSamples/tree/main/Assets/PassthroughCameraApiSamples
[LICENSE.MIT]: https://git-st.inf.tu-dresden.de/hyper/pca-apriltag/-/blob/main/LICENSE.MIT [LICENSE.MIT]: https://git-st.inf.tu-dresden.de/hyper/pca-apriltag/-/blob/main/LICENSE.MIT
[this snippet]: https://git-st.inf.tu-dresden.de/hyper/core/-/snippets/14 [this snippet]: https://git-st.inf.tu-dresden.de/hyper/core/-/snippets/14
## Attribution
- Correct Blips by CogFireStudios -- https://freesound.org/s/531510/ -- License: Creative Commons 0
...@@ -10,18 +10,46 @@ namespace Hyper.PCAAprilTag ...@@ -10,18 +10,46 @@ namespace Hyper.PCAAprilTag
{ {
public class AprilTagDetector : MonoBehaviour public class AprilTagDetector : MonoBehaviour
{ {
/// <summary>
/// Indicates whether the AprilTag detector is currently activated.
/// </summary>
public bool isActivated { get; private set; }
public static event Action<IEnumerable<TagPoseSize>> OnTagsDetected; public static event Action<IEnumerable<TagPoseSize>> OnTagsDetected;
[SerializeField] private bool isContinuous; [SerializeField, Tooltip("If true, the detector will continuously search for tags until Deactivate() is called.")]
[SerializeField] private List<IdSizePair> tagSizes = new(); private bool isContinuous;
[SerializeField] private int frameDelay = 7;
[SerializeField] private int numAveragedPoses = 10; [SerializeField, Tooltip("If true, plays a sound when tags are detected.")]
[SerializeField] private float maxLinearSpeed = 0.1f; // meters per second private bool playSuccessSound = true;
[SerializeField] private float maxAngularSpeed = 10f; // degrees per second
[SerializeField] private float timeout = 2f; [SerializeField, Tooltip("List of tag IDs and their sizes. " +
[SerializeField] private int decimation = 4; "Only tags listed here will be processed by the AprilTag detector.")]
private List<IdSizePair> tagSizes = new();
[SerializeField, Tooltip("Use the past frameDelay camera poses when detecting tags " +
"to compensate for WebCamTexture latency.")]
private int frameDelay = 7;
[SerializeField, Tooltip("Number of poses to average for each detected tag. " +
"Higher values can improve stability but may introduce latency.")]
private int numAveragedPoses = 10;
[SerializeField, Tooltip("Maximum linear speed of the headset before detection is skipped. " +
"This is to avoid bad detections when the headset is moving too fast.")]
private float maxLinearSpeed = 0.1f; // meters per second
[SerializeField, Tooltip("Maximum angular speed of the headset before detection is skipped. " +
"This is to avoid bad detections when the headset is rotating too fast.")]
private float maxAngularSpeed = 10f; // degrees per second
// [SerializeField] private float timeout = 2f; // TODO
[SerializeField, Tooltip("Decimation factor for the image processing. " +
"Higher values reduce the resolution of the input image, " +
"which can speed up processing but may reduce accuracy.")]
private int decimation = 1;
private bool _isActivated;
private float _verticalFOV; private float _verticalFOV;
private TagDetector _detector; private TagDetector _detector;
private WebCamTextureManager _webCamTextureManager; private WebCamTextureManager _webCamTextureManager;
...@@ -30,14 +58,13 @@ namespace Hyper.PCAAprilTag ...@@ -30,14 +58,13 @@ namespace Hyper.PCAAprilTag
private Thread _detectorProcessImageThread; private Thread _detectorProcessImageThread;
private List<TagPoseSize> _detectedTags = new(); private List<TagPoseSize> _detectedTags = new();
private Lifo<Pose> _cameraPoses; private Lifo<Pose> _cameraPoses;
private Dictionary<int, Lifo<TagPoseSize>> _tagPoseSizeDict = new(); private readonly Dictionary<int, Lifo<TagPoseSize>> _tagPoseSizeDict = new();
private AudioSource _doneAudioSource; private AudioSource _doneAudioSource;
private Vector3 lastPosition; private Transform _headset;
private Quaternion lastRotation; private Vector3 _lastPosition;
private float lastTime; private Quaternion _lastRotation;
private float _lastTime;
private Transform headset;
private void Awake() private void Awake()
{ {
...@@ -65,10 +92,10 @@ namespace Hyper.PCAAprilTag ...@@ -65,10 +92,10 @@ namespace Hyper.PCAAprilTag
var cameraIntrinsics = PassthroughCameraUtils.GetCameraIntrinsics(CameraEye); var cameraIntrinsics = PassthroughCameraUtils.GetCameraIntrinsics(CameraEye);
_verticalFOV = 2 * Mathf.Atan2(cameraIntrinsics.Resolution.y, 2 * cameraIntrinsics.FocalLength.y); _verticalFOV = 2 * Mathf.Atan2(cameraIntrinsics.Resolution.y, 2 * cameraIntrinsics.FocalLength.y);
headset = Camera.main!.transform; _headset = Camera.main!.transform;
lastPosition = headset.position; _lastPosition = _headset.position;
lastRotation = headset.rotation; _lastRotation = _headset.rotation;
lastTime = Time.time; _lastTime = Time.time;
} }
private void OnDestroy() private void OnDestroy()
...@@ -85,6 +112,7 @@ namespace Hyper.PCAAprilTag ...@@ -85,6 +112,7 @@ namespace Hyper.PCAAprilTag
private void LateUpdate() private void LateUpdate()
{ {
// TODO. This might be a problem if the Camera Rig is moved.
_cameraPoses.Push(PassthroughCameraUtils.GetCameraPoseInWorld(CameraEye)); _cameraPoses.Push(PassthroughCameraUtils.GetCameraPoseInWorld(CameraEye));
if (_detectedTags.Any()) if (_detectedTags.Any())
...@@ -103,44 +131,44 @@ namespace Hyper.PCAAprilTag ...@@ -103,44 +131,44 @@ namespace Hyper.PCAAprilTag
if (!isContinuous && _tagPoseSizeDict.First().Value.Count >= numAveragedPoses) if (!isContinuous && _tagPoseSizeDict.First().Value.Count >= numAveragedPoses)
{ {
Deactivate(); Deactivate();
_doneAudioSource.Play(); if (playSuccessSound) _doneAudioSource.Play();
_tagPoseSizeDict.Clear(); _tagPoseSizeDict.Clear();
} }
OnTagsDetected?.Invoke(AvgDetectedTags()); OnTagsDetected?.Invoke(AvgDetectedTags());
} }
if (!_isActivated || !_webCamTextureManager.WebCamTexture || if (!isActivated || !_webCamTextureManager.WebCamTexture ||
_detectorProcessImageThread is { IsAlive: true }) return; _detectorProcessImageThread is { IsAlive: true }) return;
var currentTime = Time.time; var currentTime = Time.time;
var deltaTime = currentTime - lastTime; var deltaTime = currentTime - _lastTime;
if (deltaTime > 0f) if (deltaTime > 0f)
{ {
// Linear velocity // Linear velocity
float linearSpeed = Vector3.Distance(headset.position, lastPosition) / deltaTime; var linearSpeed = Vector3.Distance(_headset.position, _lastPosition) / deltaTime;
// Angular velocity // Angular velocity
Quaternion deltaRotation = headset.rotation * Quaternion.Inverse(lastRotation); var deltaRotation = _headset.rotation * Quaternion.Inverse(_lastRotation);
deltaRotation.ToAngleAxis(out float angleInDegrees, out _); deltaRotation.ToAngleAxis(out float angleInDegrees, out _);
float angularSpeed = angleInDegrees / deltaTime; var angularSpeed = angleInDegrees / deltaTime;
// Update // Update
lastPosition = headset.position; _lastPosition = _headset.position;
lastRotation = headset.rotation; _lastRotation = _headset.rotation;
lastTime = currentTime; _lastTime = currentTime;
// Check thresholds // Check thresholds
if (linearSpeed > maxLinearSpeed) if (linearSpeed > maxLinearSpeed)
{ {
Debug.LogWarning($"[Camera] Headset moving too fast (linear): {linearSpeed:F2} m/s"); DebugLog.Log(LogType.Warning, $"[Camera] Headset moving too fast (linear): {linearSpeed:F2} m/s");
return; return;
} }
if (angularSpeed > maxAngularSpeed) if (angularSpeed > maxAngularSpeed)
{ {
Debug.LogWarning($"[Camera] Headset rotating too fast: {angularSpeed:F2} deg/s"); DebugLog.Log(LogType.Warning, $"[Camera] Headset rotating too fast: {angularSpeed:F2} deg/s");
return; return;
} }
} }
...@@ -160,28 +188,28 @@ namespace Hyper.PCAAprilTag ...@@ -160,28 +188,28 @@ namespace Hyper.PCAAprilTag
_detectorProcessImageThread.Start(); _detectorProcessImageThread.Start();
} }
private void ProcessImage(Color32[] image, Matrix4x4 cameraToWorldMatrix) /// <summary>
{ /// Activate the AprilTag detector and start the WebCamTexture.
_detector.ProcessImage(image, _verticalFOV, _tagSizes, cameraToWorldMatrix); /// </summary>
_detectedTags = _detector.GetDetectedTagsCopy();
}
public void Activate() public void Activate()
{ {
if (_isActivated) return; if (isActivated) return;
_isActivated = true; isActivated = true;
_webCamTextureManager.WebCamTexture.Play(); _webCamTextureManager.WebCamTexture.Play();
} }
/// <summary>
/// Deactivate the AprilTag detector and stop the WebCamTexture.
/// </summary>
public void Deactivate() public void Deactivate()
{ {
_isActivated = false; isActivated = false;
_webCamTextureManager.WebCamTexture.Stop(); _webCamTextureManager.WebCamTexture.Stop();
} }
/// <summary> /// <summary>
/// Method to wait for the camera to be ready and then activate the AprilTag detector. /// Wait for the camera to be ready and then activate the AprilTag detector.
/// Useful if AprilTag detection should run when the application starts. /// Useful if AprilTag detection should run when the application starts.
/// </summary> /// </summary>
public void WaitForCamReadyAndActivate() => public void WaitForCamReadyAndActivate() =>
...@@ -199,6 +227,72 @@ namespace Hyper.PCAAprilTag ...@@ -199,6 +227,72 @@ namespace Hyper.PCAAprilTag
} }
} }
/// <summary>
/// Add a new tag with its ID and size.
/// </summary>
/// <param name="id">Tag ID.</param>
/// <param name="size">Tag size.</param>
/// <returns>Success boolean.</returns>
public bool AddTag(int id, float size)
{
if (tagSizes.Any(pair => pair.id == id))
{
DebugLog.Log(LogType.Warning, $"Tag with ID {id} already exists.");
return false;
}
tagSizes.Add(new IdSizePair(id, size));
_tagSizes = TagSizesDictionary();
return true;
}
/// <summary>
/// Remove a tag by its ID.
/// </summary>
/// <param name="id">Tag ID.</param>
/// <returns></returns>
public bool RemoveTag(int id)
{
if (tagSizes.Any(pair => pair.id == id))
{
tagSizes.Remove(tagSizes.First(pair => pair.id == id));
_tagSizes = TagSizesDictionary();
return true;
}
DebugLog.Log(LogType.Warning, $"Tag with ID {id} does not exist.");
return false;
}
/// <summary>
/// Set the tagSizes list.
/// </summary>
/// <param name="ids">Array of tag IDs.</param>
/// <param name="sizes">Array of tag sizes.</param>
/// <returns>Success boolean.</returns>
public bool SetTagSizes(int[] ids, float[] sizes)
{
if (ids.Length != sizes.Length)
{
DebugLog.Log(LogType.Error, "IDs and sizes arrays must have the same length.");
return false;
}
tagSizes.Clear();
for (var i = 0; i < ids.Length; i++)
{
tagSizes.Add(new IdSizePair(ids[i], sizes[i]));
}
_tagSizes = TagSizesDictionary();
return true;
}
private void ProcessImage(Color32[] image, Matrix4x4 cameraToWorldMatrix)
{
_detector.ProcessImage(image, _verticalFOV, _tagSizes, cameraToWorldMatrix);
_detectedTags = _detector.GetDetectedTagsCopy();
}
private Dictionary<int, float> TagSizesDictionary() private Dictionary<int, float> TagSizesDictionary()
{ {
var dict = new Dictionary<int, float>(); var dict = new Dictionary<int, float>();
......
using Debug = UnityEngine.Debug;
using System;
using System.Diagnostics;
using UnityEngine;
namespace Hyper.PCAAprilTag
{
/// <summary>
/// A utility class for logging debug messages.
/// To enable, add "HYPER_PCA_APRILTAG_DEBUG_LOG"
/// to Project Settings > Player > Script Compilation > Scripting Define Symbols
/// </summary>
public static class DebugLog
{
[Conditional("HYPER_PCA_APRILTAG_DEBUG_LOG")]
public static void Log(LogType mType, string message)
{
switch (mType)
{
case LogType.Error:
Debug.LogError(message);
break;
case LogType.Warning:
Debug.LogWarning(message);
break;
case LogType.Log:
Debug.Log(message);
break;
default:
throw new ArgumentOutOfRangeException(nameof(mType), mType, null);
}
}
}
}
fileFormatVersion: 2 fileFormatVersion: 2
guid: 3a8ac5b7682fa4c2491fe8382784bad5 guid: 28ff872694bba47d899df394223ec8d0
\ No newline at end of file \ No newline at end of file
...@@ -3,9 +3,30 @@ using System; ...@@ -3,9 +3,30 @@ using System;
namespace Hyper.PCAAprilTag namespace Hyper.PCAAprilTag
{ {
[Serializable] [Serializable]
public struct IdSizePair public struct IdSizePair : IEquatable<IdSizePair>
{ {
public int id; public int id;
public float size; public float size;
public IdSizePair(int id, float size)
{
this.id = id;
this.size = size;
}
public bool Equals(IdSizePair other)
{
return id == other.id;
}
public override bool Equals(object obj)
{
return obj is IdSizePair other && Equals(other);
}
public override int GetHashCode()
{
return id;
}
} }
} }
using AprilTag;
using System.Collections.Generic; using System.Collections.Generic;
using Unity.Collections; using Unity.Collections;
using Unity.Mathematics; using Unity.Mathematics;
......
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
// LIFO (Last-In-First-Out) List namespace Hyper.PCAAprilTag
{
/// <summary>
/// LIFO (Last-In-First-Out) List
/// </summary>
/// <typeparam name="T"></typeparam>
public class Lifo<T> public class Lifo<T>
{ {
private readonly int _capacity; private readonly int _capacity;
...@@ -19,6 +24,7 @@ public class Lifo<T> ...@@ -19,6 +24,7 @@ public class Lifo<T>
{ {
_list.RemoveAt(0); _list.RemoveAt(0);
} }
_list.Add(item); _list.Add(item);
} }
...@@ -43,3 +49,4 @@ public class Lifo<T> ...@@ -43,3 +49,4 @@ public class Lifo<T>
public List<T> ToList() => new List<T>(_list); public List<T> ToList() => new List<T>(_list);
} }
}
// Copyright (c) Meta Platforms, Inc. and affiliates.
using UnityEngine;
public static class PassthroughCameraDebugger
{
public enum DebuglevelEnum
{
ALL,
NONE,
ONLY_ERROR,
ONLY_LOG,
ONLY_WARNING
}
public static DebuglevelEnum DebugLevel = DebuglevelEnum.ALL;
/// <summary>
/// Send debug information to Unity console based on DebugType and DebugLevel
/// </summary>
/// <param name="mType"></param>
/// <param name="message"></param>
public static void DebugMessage(LogType mType, string message)
{
switch (mType)
{
case LogType.Error:
if (DebugLevel is DebuglevelEnum.ALL or DebuglevelEnum.ONLY_ERROR)
{
Debug.LogError(message);
}
break;
case LogType.Log:
if (DebugLevel is DebuglevelEnum.ALL or DebuglevelEnum.ONLY_LOG)
{
Debug.Log(message);
}
break;
case LogType.Warning:
if (DebugLevel is DebuglevelEnum.ALL or DebuglevelEnum.ONLY_WARNING)
{
Debug.LogWarning(message);
}
break;
}
}
}
...@@ -5,17 +5,19 @@ using System.Linq; ...@@ -5,17 +5,19 @@ using System.Linq;
using UnityEngine; using UnityEngine;
#if UNITY_ANDROID #if UNITY_ANDROID
using UnityEngine.Android; using UnityEngine.Android;
using PCD = PassthroughCameraDebugger;
#pragma warning disable CS0618 // Type or member is obsolete #pragma warning disable CS0618 // Type or member is obsolete
#endif #endif
namespace Hyper.PCAAprilTag
{
/// <summary> /// <summary>
/// PLEASE NOTE: Unity doesn't support requesting multiple permissions at the same time with <see cref="Permission.RequestUserPermissions"/> on Android. /// PLEASE NOTE: Unity doesn't support requesting multiple permissions at the same time with <see cref="Permission.RequestUserPermissions"/> on Android.
/// This component is a sample and shouldn't be used simultaneously with other scripts that manage Android permissions. /// This component is a sample and shouldn't be used simultaneously with other scripts that manage Android permissions.
/// </summary> /// </summary>
public class PassthroughCameraPermissions : MonoBehaviour public class PassthroughCameraPermissions : MonoBehaviour
{ {
[SerializeField] public List<string> PermissionRequestsOnStartup = new() { OVRPermissionsRequester.ScenePermission }; [SerializeField]
public List<string> PermissionRequestsOnStartup = new() { OVRPermissionsRequester.ScenePermission };
public static readonly string[] CameraPermissions = public static readonly string[] CameraPermissions =
{ {
...@@ -36,15 +38,16 @@ public class PassthroughCameraPermissions : MonoBehaviour ...@@ -36,15 +38,16 @@ public class PassthroughCameraPermissions : MonoBehaviour
{ {
return; return;
} }
s_askedOnce = true; s_askedOnce = true;
if (IsAllCameraPermissionsGranted()) if (IsAllCameraPermissionsGranted())
{ {
HasCameraPermission = true; HasCameraPermission = true;
PCD.DebugMessage(LogType.Log, "PCA: All camera permissions granted."); DebugLog.Log(LogType.Log, "PCA: All camera permissions granted.");
} }
else else
{ {
PCD.DebugMessage(LogType.Log, "PCA: Requesting camera permissions."); DebugLog.Log(LogType.Log, "PCA: Requesting camera permissions.");
var callbacks = new PermissionCallbacks(); var callbacks = new PermissionCallbacks();
callbacks.PermissionDenied += PermissionCallbacksPermissionDenied; callbacks.PermissionDenied += PermissionCallbacksPermissionDenied;
...@@ -63,7 +66,7 @@ public class PassthroughCameraPermissions : MonoBehaviour ...@@ -63,7 +66,7 @@ public class PassthroughCameraPermissions : MonoBehaviour
/// <param name="permissionName"></param> /// <param name="permissionName"></param>
private static void PermissionCallbacksPermissionGranted(string permissionName) private static void PermissionCallbacksPermissionGranted(string permissionName)
{ {
PCD.DebugMessage(LogType.Log, $"PCA: Permission {permissionName} Granted"); DebugLog.Log(LogType.Log, $"PCA: Permission {permissionName} Granted");
// Only initialize the WebCamTexture object if both permissions are granted // Only initialize the WebCamTexture object if both permissions are granted
if (IsAllCameraPermissionsGranted()) if (IsAllCameraPermissionsGranted())
...@@ -78,11 +81,13 @@ public class PassthroughCameraPermissions : MonoBehaviour ...@@ -78,11 +81,13 @@ public class PassthroughCameraPermissions : MonoBehaviour
/// <param name="permissionName"></param> /// <param name="permissionName"></param>
private static void PermissionCallbacksPermissionDenied(string permissionName) private static void PermissionCallbacksPermissionDenied(string permissionName)
{ {
PCD.DebugMessage(LogType.Warning, $"PCA: Permission {permissionName} Denied"); DebugLog.Log(LogType.Warning, $"PCA: Permission {permissionName} Denied");
HasCameraPermission = false; HasCameraPermission = false;
s_askedOnce = false; s_askedOnce = false;
} }
private static bool IsAllCameraPermissionsGranted() => CameraPermissions.All(Permission.HasUserAuthorizedPermission); private static bool IsAllCameraPermissionsGranted() =>
CameraPermissions.All(Permission.HasUserAuthorizedPermission);
#endif #endif
} }
}
...@@ -6,6 +6,8 @@ using System.Collections.Generic; ...@@ -6,6 +6,8 @@ using System.Collections.Generic;
using UnityEngine; using UnityEngine;
using UnityEngine.Assertions; using UnityEngine.Assertions;
namespace Hyper.PCAAprilTag
{
public static class PassthroughCameraUtils public static class PassthroughCameraUtils
{ {
// The Horizon OS starts supporting PCA with v74. // The Horizon OS starts supporting PCA with v74.
...@@ -20,8 +22,12 @@ public static class PassthroughCameraUtils ...@@ -20,8 +22,12 @@ public static class PassthroughCameraUtils
private static int? s_horizonOsVersion; private static int? s_horizonOsVersion;
// Caches // Caches
internal static readonly Dictionary<PassthroughCameraEye, (string id, int index)> CameraEyeToCameraIdMap = new(); internal static readonly Dictionary<PassthroughCameraEye, (string id, int index)>
private static readonly ConcurrentDictionary<PassthroughCameraEye, List<Vector2Int>> s_cameraOutputSizes = new(); CameraEyeToCameraIdMap = new();
private static readonly ConcurrentDictionary<PassthroughCameraEye, List<Vector2Int>>
s_cameraOutputSizes = new();
private static readonly ConcurrentDictionary<string, AndroidJavaObject> s_cameraCharacteristicsMap = new(); private static readonly ConcurrentDictionary<string, AndroidJavaObject> s_cameraCharacteristicsMap = new();
private static readonly OVRPose?[] s_cachedCameraPosesRelativeToHead = new OVRPose?[2]; private static readonly OVRPose?[] s_cachedCameraPosesRelativeToHead = new OVRPose?[2];
...@@ -92,7 +98,8 @@ public static class PassthroughCameraUtils ...@@ -92,7 +98,8 @@ public static class PassthroughCameraUtils
// Querying the camera resolution for which the intrinsics are provided // Querying the camera resolution for which the intrinsics are provided
// https://developer.android.com/reference/android/hardware/camera2/CameraCharacteristics#SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE // https://developer.android.com/reference/android/hardware/camera2/CameraCharacteristics#SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE
// This is a Rect of 4 elements: [bottom, left, right, top] with (0,0) at top-left corner. // This is a Rect of 4 elements: [bottom, left, right, top] with (0,0) at top-left corner.
using var sensorSize = GetCameraValueByKey<AndroidJavaObject>(cameraCharacteristics, "SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE"); using var sensorSize = GetCameraValueByKey<AndroidJavaObject>(cameraCharacteristics,
"SENSOR_INFO_PRE_CORRECTION_ACTIVE_ARRAY_SIZE");
return new PassthroughCameraIntrinsics return new PassthroughCameraIntrinsics
{ {
...@@ -130,13 +137,15 @@ public static class PassthroughCameraUtils ...@@ -130,13 +137,15 @@ public static class PassthroughCameraUtils
if (s_cachedCameraPosesRelativeToHead[index] == null) if (s_cachedCameraPosesRelativeToHead[index] == null)
{ {
var cameraId = GetCameraIdByEye(cameraEye); var cameraId = GetCameraIdByEye(cameraEye);
using var cameraCharacteristics = s_cameraManager.Call<AndroidJavaObject>("getCameraCharacteristics", cameraId); using var cameraCharacteristics =
s_cameraManager.Call<AndroidJavaObject>("getCameraCharacteristics", cameraId);
var cameraTranslation = GetCameraValueByKey<float[]>(cameraCharacteristics, "LENS_POSE_TRANSLATION"); var cameraTranslation = GetCameraValueByKey<float[]>(cameraCharacteristics, "LENS_POSE_TRANSLATION");
var p_headFromCamera = new Vector3(cameraTranslation[0], cameraTranslation[1], -cameraTranslation[2]); var p_headFromCamera = new Vector3(cameraTranslation[0], cameraTranslation[1], -cameraTranslation[2]);
var cameraRotation = GetCameraValueByKey<float[]>(cameraCharacteristics, "LENS_POSE_ROTATION"); var cameraRotation = GetCameraValueByKey<float[]>(cameraCharacteristics, "LENS_POSE_ROTATION");
var q_cameraFromHead = new Quaternion(-cameraRotation[0], -cameraRotation[1], cameraRotation[2], cameraRotation[3]); var q_cameraFromHead = new Quaternion(-cameraRotation[0], -cameraRotation[1], cameraRotation[2],
cameraRotation[3]);
var q_headFromCamera = Quaternion.Inverse(q_cameraFromHead); var q_headFromCamera = Quaternion.Inverse(q_cameraFromHead);
...@@ -203,14 +212,14 @@ public static class PassthroughCameraUtils ...@@ -203,14 +212,14 @@ public static class PassthroughCameraUtils
return true; return true;
} }
Debug.Log($"PCA: PassthroughCamera - Initializing..."); DebugLog.Log(LogType.Log, $"PCA: PassthroughCamera - Initializing...");
using var activityClass = new AndroidJavaClass("com.unity3d.player.UnityPlayer"); using var activityClass = new AndroidJavaClass("com.unity3d.player.UnityPlayer");
s_currentActivity = activityClass.GetStatic<AndroidJavaObject>("currentActivity"); s_currentActivity = activityClass.GetStatic<AndroidJavaObject>("currentActivity");
s_cameraManager = s_currentActivity.Call<AndroidJavaObject>("getSystemService", "camera"); s_cameraManager = s_currentActivity.Call<AndroidJavaObject>("getSystemService", "camera");
Assert.IsNotNull(s_cameraManager, "Camera manager has not been provided by the Android system"); Assert.IsNotNull(s_cameraManager, "Camera manager has not been provided by the Android system");
var cameraIds = GetCameraIdList(); var cameraIds = GetCameraIdList();
Debug.Log($"PCA: PassthroughCamera - cameraId list is {string.Join(", ", cameraIds)}"); DebugLog.Log(LogType.Log, $"PCA: PassthroughCamera - cameraId list is {string.Join(", ", cameraIds)}");
for (var idIndex = 0; idIndex < cameraIds.Length; idIndex++) for (var idIndex = 0; idIndex < cameraIds.Length; idIndex++)
{ {
...@@ -226,7 +235,8 @@ public static class PassthroughCameraUtils ...@@ -226,7 +235,8 @@ public static class PassthroughCameraUtils
using var key = keysList.Call<AndroidJavaObject>("get", i); using var key = keysList.Call<AndroidJavaObject>("get", i);
var keyName = key.Call<string>("getName"); var keyName = key.Call<string>("getName");
if (string.Equals(keyName, "com.meta.extra_metadata.camera_source", StringComparison.OrdinalIgnoreCase)) if (string.Equals(keyName, "com.meta.extra_metadata.camera_source",
StringComparison.OrdinalIgnoreCase))
{ {
// Both `com.meta.extra_metadata.camera_source` and `com.meta.extra_metadata.camera_source` are // Both `com.meta.extra_metadata.camera_source` and `com.meta.extra_metadata.camera_source` are
// custom camera fields which are stored as arrays of size 1, instead of single values. // custom camera fields which are stored as arrays of size 1, instead of single values.
...@@ -237,7 +247,8 @@ public static class PassthroughCameraUtils ...@@ -237,7 +247,8 @@ public static class PassthroughCameraUtils
cameraSource = (CameraSource)cameraSourceArr[0]; cameraSource = (CameraSource)cameraSourceArr[0];
} }
else if (string.Equals(keyName, "com.meta.extra_metadata.position", StringComparison.OrdinalIgnoreCase)) else if (string.Equals(keyName, "com.meta.extra_metadata.position",
StringComparison.OrdinalIgnoreCase))
{ {
var cameraPositionArr = GetCameraValueByKey<sbyte[]>(cameraCharacteristics, key); var cameraPositionArr = GetCameraValueByKey<sbyte[]>(cameraCharacteristics, key);
if (cameraPositionArr == null || cameraPositionArr.Length != 1) if (cameraPositionArr == null || cameraPositionArr.Length != 1)
...@@ -247,17 +258,18 @@ public static class PassthroughCameraUtils ...@@ -247,17 +258,18 @@ public static class PassthroughCameraUtils
} }
} }
if (!cameraSource.HasValue || !cameraPosition.HasValue || cameraSource.Value != CameraSource.Passthrough) if (!cameraSource.HasValue || !cameraPosition.HasValue ||
cameraSource.Value != CameraSource.Passthrough)
continue; continue;
switch (cameraPosition) switch (cameraPosition)
{ {
case CameraPosition.Left: case CameraPosition.Left:
Debug.Log($"PCA: Found left passthrough cameraId = {cameraId}"); DebugLog.Log(LogType.Log, $"PCA: Found left passthrough cameraId = {cameraId}");
CameraEyeToCameraIdMap[PassthroughCameraEye.Left] = (cameraId, idIndex); CameraEyeToCameraIdMap[PassthroughCameraEye.Left] = (cameraId, idIndex);
break; break;
case CameraPosition.Right: case CameraPosition.Right:
Debug.Log($"PCA: Found right passthrough cameraId = {cameraId}"); DebugLog.Log(LogType.Log, $"PCA: Found right passthrough cameraId = {cameraId}");
CameraEyeToCameraIdMap[PassthroughCameraEye.Right] = (cameraId, idIndex); CameraEyeToCameraIdMap[PassthroughCameraEye.Right] = (cameraId, idIndex);
break; break;
default: default:
...@@ -352,16 +364,20 @@ public struct PassthroughCameraIntrinsics ...@@ -352,16 +364,20 @@ public struct PassthroughCameraIntrinsics
/// The focal length in pixels /// The focal length in pixels
/// </summary> /// </summary>
public Vector2 FocalLength; public Vector2 FocalLength;
/// <summary> /// <summary>
/// The principal point from the top-left corner of the image, expressed in pixels /// The principal point from the top-left corner of the image, expressed in pixels
/// </summary> /// </summary>
public Vector2 PrincipalPoint; public Vector2 PrincipalPoint;
/// <summary> /// <summary>
/// The resolution in pixels for which the intrinsics are defined /// The resolution in pixels for which the intrinsics are defined
/// </summary> /// </summary>
public Vector2Int Resolution; public Vector2Int Resolution;
/// <summary> /// <summary>
/// The skew coefficient which represents the non-perpendicularity of the image sensor's x and y axes /// The skew coefficient which represents the non-perpendicularity of the image sensor's x and y axes
/// </summary> /// </summary>
public float Skew; public float Skew;
} }
}
...@@ -4,14 +4,18 @@ using System.Collections; ...@@ -4,14 +4,18 @@ using System.Collections;
using System.Linq; using System.Linq;
using UnityEngine; using UnityEngine;
using UnityEngine.Assertions; using UnityEngine.Assertions;
using PCD = PassthroughCameraDebugger;
namespace Hyper.PCAAprilTag
{
public class WebCamTextureManager : MonoBehaviour public class WebCamTextureManager : MonoBehaviour
{ {
[SerializeField] public PassthroughCameraEye Eye = PassthroughCameraEye.Left; [SerializeField] public PassthroughCameraEye Eye = PassthroughCameraEye.Left;
[SerializeField, Tooltip("The requested resolution of the camera may not be supported by the chosen camera. In such cases, the closest available values will be used.\n\n" +
[SerializeField, Tooltip(
"The requested resolution of the camera may not be supported by the chosen camera. In such cases, the closest available values will be used.\n\n" +
"When set to (0,0), the highest supported resolution will be used.")] "When set to (0,0), the highest supported resolution will be used.")]
public Vector2Int RequestedResolution; public Vector2Int RequestedResolution;
[SerializeField] public PassthroughCameraPermissions CameraPermissions; [SerializeField] public PassthroughCameraPermissions CameraPermissions;
/// <summary> /// <summary>
...@@ -25,8 +29,9 @@ public class WebCamTextureManager : MonoBehaviour ...@@ -25,8 +29,9 @@ public class WebCamTextureManager : MonoBehaviour
private void Awake() private void Awake()
{ {
PCD.DebugMessage(LogType.Log, $"{nameof(WebCamTextureManager)}.{nameof(Awake)}() was called"); DebugLog.Log(LogType.Log, $"{nameof(WebCamTextureManager)}.{nameof(Awake)}() was called");
Assert.AreEqual(1, FindObjectsByType<WebCamTextureManager>(FindObjectsInactive.Include, FindObjectsSortMode.None).Length, Assert.AreEqual(1,
FindObjectsByType<WebCamTextureManager>(FindObjectsInactive.Include, FindObjectsSortMode.None).Length,
$"PCA: Passthrough Camera: more than one {nameof(WebCamTextureManager)} component. Only one instance is allowed at a time. Current instance: {name}"); $"PCA: Passthrough Camera: more than one {nameof(WebCamTextureManager)} component. Only one instance is allowed at a time. Current instance: {name}");
#if UNITY_ANDROID #if UNITY_ANDROID
CameraPermissions.AskCameraPermissions(); CameraPermissions.AskCameraPermissions();
...@@ -35,10 +40,11 @@ public class WebCamTextureManager : MonoBehaviour ...@@ -35,10 +40,11 @@ public class WebCamTextureManager : MonoBehaviour
private void OnEnable() private void OnEnable()
{ {
PCD.DebugMessage(LogType.Log, $"PCA: {nameof(OnEnable)}() was called"); DebugLog.Log(LogType.Log, $"PCA: {nameof(OnEnable)}() was called");
if (!PassthroughCameraUtils.IsSupported) if (!PassthroughCameraUtils.IsSupported)
{ {
PCD.DebugMessage(LogType.Log, "PCA: Passthrough Camera functionality is not supported by the current device." + DebugLog.Log(LogType.Log,
"PCA: Passthrough Camera functionality is not supported by the current device." +
$" Disabling {nameof(WebCamTextureManager)} object"); $" Disabling {nameof(WebCamTextureManager)} object");
enabled = false; enabled = false;
return; return;
...@@ -47,18 +53,18 @@ public class WebCamTextureManager : MonoBehaviour ...@@ -47,18 +53,18 @@ public class WebCamTextureManager : MonoBehaviour
m_hasPermission = PassthroughCameraPermissions.HasCameraPermission == true; m_hasPermission = PassthroughCameraPermissions.HasCameraPermission == true;
if (!m_hasPermission) if (!m_hasPermission)
{ {
PCD.DebugMessage(LogType.Error, DebugLog.Log(LogType.Error,
$"PCA: Passthrough Camera requires permission(s) {string.Join(" and ", PassthroughCameraPermissions.CameraPermissions)}. Waiting for them to be granted..."); $"PCA: Passthrough Camera requires permission(s) {string.Join(" and ", PassthroughCameraPermissions.CameraPermissions)}. Waiting for them to be granted...");
return; return;
} }
PCD.DebugMessage(LogType.Log, "PCA: All permissions have been granted"); DebugLog.Log(LogType.Log, "PCA: All permissions have been granted");
_ = StartCoroutine(InitializeWebCamTexture()); _ = StartCoroutine(InitializeWebCamTexture());
} }
private void OnDisable() private void OnDisable()
{ {
PCD.DebugMessage(LogType.Log, $"PCA: {nameof(OnDisable)}() was called"); DebugLog.Log(LogType.Log, $"PCA: {nameof(OnDisable)}() was called");
StopCoroutine(InitializeWebCamTexture()); StopCoroutine(InitializeWebCamTexture());
if (WebCamTexture != null) if (WebCamTexture != null)
{ {
...@@ -86,7 +92,7 @@ public class WebCamTextureManager : MonoBehaviour ...@@ -86,7 +92,7 @@ public class WebCamTextureManager : MonoBehaviour
var ptLayer = FindAnyObjectByType<OVRPassthroughLayer>(); var ptLayer = FindAnyObjectByType<OVRPassthroughLayer>();
if (ptLayer == null || !PassthroughCameraUtils.IsPassthroughEnabled()) if (ptLayer == null || !PassthroughCameraUtils.IsPassthroughEnabled())
{ {
PCD.DebugMessage(LogType.Error, "Passthrough must be enabled to use the Passthrough Camera API."); DebugLog.Log(LogType.Error, "Passthrough must be enabled to use the Passthrough Camera API.");
yield break; yield break;
} }
...@@ -99,7 +105,8 @@ public class WebCamTextureManager : MonoBehaviour ...@@ -99,7 +105,8 @@ public class WebCamTextureManager : MonoBehaviour
while (true) while (true)
{ {
var devices = WebCamTexture.devices; var devices = WebCamTexture.devices;
if (PassthroughCameraUtils.EnsureInitialized() && PassthroughCameraUtils.CameraEyeToCameraIdMap.TryGetValue(Eye, out var cameraData)) if (PassthroughCameraUtils.EnsureInitialized() &&
PassthroughCameraUtils.CameraEyeToCameraIdMap.TryGetValue(Eye, out var cameraData))
{ {
if (cameraData.index < devices.Length) if (cameraData.index < devices.Length)
{ {
...@@ -107,21 +114,26 @@ public class WebCamTextureManager : MonoBehaviour ...@@ -107,21 +114,26 @@ public class WebCamTextureManager : MonoBehaviour
WebCamTexture webCamTexture; WebCamTexture webCamTexture;
if (RequestedResolution == Vector2Int.zero) if (RequestedResolution == Vector2Int.zero)
{ {
var largestResolution = PassthroughCameraUtils.GetOutputSizes(Eye).OrderBy(static size => size.x * size.y).Last(); var largestResolution = PassthroughCameraUtils.GetOutputSizes(Eye)
.OrderBy(static size => size.x * size.y).Last();
webCamTexture = new WebCamTexture(deviceName, largestResolution.x, largestResolution.y); webCamTexture = new WebCamTexture(deviceName, largestResolution.x, largestResolution.y);
} }
else else
{ {
webCamTexture = new WebCamTexture(deviceName, RequestedResolution.x, RequestedResolution.y); webCamTexture = new WebCamTexture(deviceName, RequestedResolution.x, RequestedResolution.y);
} }
webCamTexture.Play(); webCamTexture.Play();
var currentResolution = new Vector2Int(webCamTexture.width, webCamTexture.height); var currentResolution = new Vector2Int(webCamTexture.width, webCamTexture.height);
if (RequestedResolution != Vector2Int.zero && RequestedResolution != currentResolution) if (RequestedResolution != Vector2Int.zero && RequestedResolution != currentResolution)
{ {
PCD.DebugMessage(LogType.Warning, $"WebCamTexture created, but '{nameof(RequestedResolution)}' {RequestedResolution} is not supported. Current resolution: {currentResolution}."); DebugLog.Log(LogType.Warning,
$"WebCamTexture created, but '{nameof(RequestedResolution)}' {RequestedResolution} is not supported. Current resolution: {currentResolution}.");
} }
WebCamTexture = webCamTexture; WebCamTexture = webCamTexture;
PCD.DebugMessage(LogType.Log, $"WebCamTexture created, texturePtr: {WebCamTexture.GetNativeTexturePtr()}, size: {WebCamTexture.width}/{WebCamTexture.height}"); DebugLog.Log(LogType.Log,
$"WebCamTexture created, texturePtr: {WebCamTexture.GetNativeTexturePtr()}, size: {WebCamTexture.width}/{WebCamTexture.height}");
webCamTexture.Stop(); webCamTexture.Stop();
IsReady = true; IsReady = true;
...@@ -130,7 +142,8 @@ public class WebCamTextureManager : MonoBehaviour ...@@ -130,7 +142,8 @@ public class WebCamTextureManager : MonoBehaviour
} }
} }
PCD.DebugMessage(LogType.Error, $"Requested camera is not present in WebCamTexture.devices: {string.Join(", ", devices)}."); DebugLog.Log(LogType.Error,
$"Requested camera is not present in WebCamTexture.devices: {string.Join(", ", devices)}.");
yield return null; yield return null;
} }
} }
...@@ -144,3 +157,4 @@ public enum PassthroughCameraEye ...@@ -144,3 +157,4 @@ public enum PassthroughCameraEye
Left, Left,
Right Right
} }
}
{ {
"name": "de.tu-dresden.hyper.pca-apriltag", "name": "de.tu-dresden.hyper.pca-apriltag",
"version": "1.0.0", "version": "1.0.1",
"displayName": "HYPER PCA AprilTag", "displayName": "HYPER PCA AprilTag",
"description": "This package provides AprilTag detection using Passthrough Camera API.", "description": "This package provides AprilTag detection using Passthrough Camera API.",
"unity": "6000.0", "unity": "6000.0",
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment