diff --git a/Assets/AXIP/AILIA-MODELS/ImageCaptioning.meta b/Assets/AXIP/AILIA-MODELS/ImageCaptioning.meta new file mode 100644 index 00000000..69c669cf --- /dev/null +++ b/Assets/AXIP/AILIA-MODELS/ImageCaptioning.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 55616d136d3f8f8438a986d29b43af1e +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaCaptioning.cs b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaCaptioning.cs new file mode 100644 index 00000000..78055251 --- /dev/null +++ b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaCaptioning.cs @@ -0,0 +1,274 @@ +using ailiaSDK; +using System; +using System.Collections; +using System.IO; +using System.Linq; +using UnityEngine; + +struct JsonVocabulary +{ + public string[] vocab; +} + +public class AiliaCaptioning : IDisposable +{ + private AiliaDownload ailiaDownload; + + private AiliaModel ailiaCaptioningFeatures = new AiliaModel(); + private AiliaModel ailiaCaptioningInference = new AiliaModel(); + + private string[] jsonDict; + + public bool gpuMode { get; private set; } + + public AiliaCaptioning(AiliaDownload ailiaDownload = null) + { + this.ailiaDownload = ailiaDownload ?? new AiliaDownload(); + } + + public IEnumerator Initialize(bool gpuMode, Action onCompleted) + { + this.gpuMode = gpuMode; + string assetPath = Application.temporaryCachePath; + string folderPath = "image_captioning_pytorch"; + string featuresModelName = "model_feat"; + string inferenceModelName = "model_fc"; + + // captioning feature extraction + if (gpuMode) + { + ailiaCaptioningFeatures.Environment(Ailia.AILIA_ENVIRONMENT_TYPE_GPU); + } + + yield return ailiaDownload.DownloadWithProgressFromURL(new ModelDownloadURL[] { + new ModelDownloadURL() { folder_path = folderPath, file_name = $"{featuresModelName}.onnx.prototxt" }, + new ModelDownloadURL() { folder_path = folderPath, file_name = $"{featuresModelName}.onnx" }, + new ModelDownloadURL() { folder_path = folderPath, file_name = $"{inferenceModelName}.onnx.prototxt" }, + new ModelDownloadURL() { folder_path = folderPath, file_name = $"{inferenceModelName}.onnx" } + }.ToList(), () => + { + bool status = false; + + status = ailiaCaptioningFeatures.OpenFile($"{assetPath}/{featuresModelName}.onnx.prototxt", $"{assetPath}/{featuresModelName}.onnx"); + + if (status == false) + { + Debug.LogError($"Could not load model {featuresModelName} at '{assetPath}': {ailiaCaptioningFeatures.GetErrorDetail()}"); + onCompleted(false); + return; + } + + status = ailiaCaptioningInference.OpenFile($"{assetPath}/{inferenceModelName}.onnx.prototxt", $"{assetPath}/{inferenceModelName}.onnx"); + + if (status == false) + { + Debug.LogError($"Could not load model {inferenceModelName} at '{assetPath}': {ailiaCaptioningInference.GetErrorDetail()}"); + onCompleted(false); + return; + } + + try + { + string dictJSON = File.ReadAllText(Application.streamingAssetsPath + "/AILIA/captioning_vocab.json"); + jsonDict = JsonUtility.FromJson("{ \"vocab\": " + dictJSON + " }").vocab; + } + catch (Exception e) + { + Debug.LogError($"Could not load the captioning vocabulary from JSON file: {e.Message}"); + onCompleted(false); + return; + } + + onCompleted(true); + return; + }); + } + + private string RunCaptioningInferenceFromFeatures(float[] features) + { + bool status; + string assetPath = Application.streamingAssetsPath + "/AILIA"; + + int inputBlobIndex = ailiaCaptioningInference.FindBlobIndexByName("fc_feats"); + + status = ailiaCaptioningInference.SetInputBlobData(features, inputBlobIndex); + + if (status == false) + { + Debug.LogError("Could not set input blob data"); + return ""; + } + + bool result = ailiaCaptioningInference.Update(); + + if (result == false) + { + Debug.Log(ailiaCaptioningInference.GetErrorDetail()); + } + + Debug.Log("[F] RAN TO COMPLETION, result: " + result); + + int outputBlobIndex = ailiaCaptioningInference.FindBlobIndexByName("seq"); + float[] outputArray = new float[20]; + + status = ailiaCaptioningInference.GetBlobData(outputArray, outputBlobIndex); + + if (status == false) + { + Debug.LogError("Could not get output blob data " + outputBlobIndex); + Debug.LogError(ailiaCaptioningInference.GetErrorDetail()); + return ""; + } + + Debug.Log(String.Join(" ", outputArray)); + int firstNullWordIndex = Array.FindIndex(outputArray, i => i == 0); + firstNullWordIndex = (firstNullWordIndex < 0 ? outputArray.Length : firstNullWordIndex); + return String.Join(" ", outputArray.Take(firstNullWordIndex).Select(i => (i > 0 ? jsonDict[((int)i) - 1] : "")).ToArray()); + } + + private float[] inputArray; + private uint channelCount = 3; + private uint featureCount = 2048; + private uint inputWidth; + private uint inputHeight; + public void PreprocessTexture(Texture2D texture) + { + int maxInputWidth = 640; + + if (texture.width > maxInputWidth) + { + RenderTexture previousActiveTexture = RenderTexture.active; + int newWidth = maxInputWidth; + int newHeight = ((int)texture.height * maxInputWidth / texture.width); + Vector2 scale = new Vector2((float)newWidth / texture.width, (float)newHeight / texture.height); + Texture2D resizedTexture = new Texture2D(newWidth, newHeight, texture.format, false); + RenderTexture.active = new RenderTexture(resizedTexture.width, resizedTexture.height, 32); + Graphics.Blit(texture, RenderTexture.active, Vector2.one, Vector2.zero); + resizedTexture.ReadPixels(new Rect { x = 0, y = 0, width = resizedTexture.width, height = resizedTexture.height }, 0, 0); + resizedTexture.Apply(); + RenderTexture.active = previousActiveTexture; + + texture = resizedTexture; + } + + + inputWidth = ((uint)texture.width); + inputHeight = ((uint)texture.height); + + float[] normalizedMean = new float[] { 0.485f, 0.456f, 0.406f }; + float[] normalizedStandardDeviation = new float[] { 0.229f, 0.224f, 0.225f }; + Color32[] colorData = texture.GetPixels32(); + + Debug.Log("SIZE " + inputWidth + " " + inputHeight); + + inputArray = new float[inputWidth * inputHeight * channelCount]; + + for (int heightIndex = 0; heightIndex < inputHeight; heightIndex++) + { + for (int widthIndex = 0; widthIndex < inputWidth; widthIndex++) + { + Color32 value = colorData[heightIndex * inputWidth + widthIndex]; + + inputArray[((0 * inputHeight + heightIndex) * inputWidth) + widthIndex] = (((float)value.r) / 255 - normalizedMean[0]) / normalizedStandardDeviation[0]; + inputArray[((1 * inputHeight + heightIndex) * inputWidth) + widthIndex] = (((float)value.g) / 255 - normalizedMean[1]) / normalizedStandardDeviation[1]; + inputArray[((2 * inputHeight + heightIndex) * inputWidth) + widthIndex] = (((float)value.b) / 255 - normalizedMean[2]) / normalizedStandardDeviation[2]; + } + } + } + + public string InferCaptionFromFrame() + { + bool status; + float[] outputArray = new float[featureCount]; + + int inputBlobIndex = ailiaCaptioningFeatures.FindBlobIndexByName("img"); + + status = ailiaCaptioningFeatures.SetInputBlobShape( + new Ailia.AILIAShape + { + x = inputWidth, + y = inputHeight, + z = channelCount, + w = 1, + dim = 3 + }, + inputBlobIndex + ); + + if (status == false) + { + Debug.LogError("Could not set input blob shape"); + return ""; + } + + status = ailiaCaptioningFeatures.SetInputBlobData(inputArray, inputBlobIndex); + + if (status == false) + { + Debug.LogError("Could not set input blob data"); + return ""; + } + + Debug.Log($"INPUT\n{inputArray[0]} {inputArray[1]} {inputArray[2]} {inputArray[3]} {inputArray[4]}"); + + bool result = ailiaCaptioningFeatures.Update(); + + if (result == false) + { + Debug.Log(ailiaCaptioningFeatures.GetErrorDetail()); + } + + Debug.Log("RAN TO COMPLETION, result: " + result); + + int outputBlobIndex = ailiaCaptioningFeatures.FindBlobIndexByName("fc"); + + status = ailiaCaptioningFeatures.GetBlobData(outputArray, outputBlobIndex); + + if (status == false) + { + Debug.LogError("Could not get output blob data " + outputBlobIndex); + Debug.LogError(ailiaCaptioningFeatures.GetErrorDetail()); + return ""; + } + + return RunCaptioningInferenceFromFeatures(outputArray); + } + +#region IDisposable Support + private bool disposedValue = false; // To detect redundant calls + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + ailiaCaptioningFeatures.Close(); + ailiaCaptioningInference.Close(); + ailiaCaptioningFeatures = null; + ailiaCaptioningInference = null; + } + + disposedValue = true; + } + } + + // TODO: override a finalizer only if Dispose(bool disposing) above has code to free unmanaged resources. + ~AiliaCaptioning() + { + // Do not change this code. Put cleanup code in Dispose(bool disposing) above. + Dispose(false); + } + + // This code added to correctly implement the disposable pattern. + public void Dispose() + { + // Do not change this code. Put cleanup code in Dispose(bool disposing) above. + Dispose(true); + // TODO: uncomment the following line if the finalizer is overridden above. + GC.SuppressFinalize(this); + } +#endregion + + +} diff --git a/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaCaptioning.cs.meta b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaCaptioning.cs.meta new file mode 100644 index 00000000..21c7fa13 --- /dev/null +++ b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaCaptioning.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d60dae19edf4dfc4ab38d5d87bc9f16a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaImageCaptioningSample.cs b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaImageCaptioningSample.cs new file mode 100644 index 00000000..908214c8 --- /dev/null +++ b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaImageCaptioningSample.cs @@ -0,0 +1,134 @@ +/* AILIA Unity Plugin Classifier Sample */ +/* Copyright 2018-2019 AXELL CORPORATION */ + +using System.Collections; +using System.Collections.Generic; +using System; +using System.IO; +using System.Runtime.InteropServices; + +using UnityEngine; +using UnityEngine.UI; + +namespace ailiaSDK +{ + public class AiliaImageCaptioningSample : AiliaRenderer + { + [SerializeField] + private GameObject UICanvas = null; + + //Settings + [SerializeField] + private bool gpu_mode = true; + [SerializeField] + private bool is_english = false; + [SerializeField] + private int camera_id = 0; + + //Output buffer + public RawImage raw_image = null; + public Text mode_text = null; + public Text label_text = null; + + //Preview texture + private Texture2D preview_texture = null; + + //ailia Instance + private AiliaCamera ailia_camera = new AiliaCamera(); + private AiliaDownload ailia_download = new AiliaDownload(); + + private AiliaCaptioning ailiaCaptioning; + + // AILIA open file + private bool FileOpened = false; + + private void CreateAilia() + { + string asset_path = Application.temporaryCachePath; + var urlList = new List(); + + ailiaCaptioning = new AiliaCaptioning(ailia_download); + StartCoroutine(ailiaCaptioning.Initialize(gpu_mode, (success) => FileOpened = success)); + } + + private void DestroyAilia() + { + ailiaCaptioning.Dispose(); + } + + void Start() + { + mode_text.text = "ailia Image captioning"; + SetUIProperties(); + CreateAilia(); + ailia_camera.CreateCamera(camera_id); + } + + void Update() + { + if (!ailia_camera.IsEnable() || !FileOpened) + { + return; + } + + //Clear label + Clear(); + + //Get camera image + int tex_width = ailia_camera.GetWidth(); + int tex_height = ailia_camera.GetHeight(); + if (preview_texture == null) + { + preview_texture = new Texture2D(tex_width, tex_height); + raw_image.texture = preview_texture; + } + Color32[] camera = ailia_camera.GetPixels32(); + + //Classify + long start_time = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond; + ailiaCaptioning.PreprocessTexture(ailia_camera.GetTexture2D()); + string caption = ailiaCaptioning.InferCaptionFromFrame(); + long end_time = DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond; ; + + //Display prediction time + if (label_text != null) + { + label_text.text = (end_time - start_time) + "ms\n" + caption; + } + + //Apply image + preview_texture.SetPixels32(camera); + preview_texture.Apply(); + } + + void SetUIProperties() + { + if (UICanvas == null) return; + // Set up UI for AiliaDownloader + var downloaderProgressPanel = UICanvas.transform.Find("DownloaderProgressPanel"); + ailia_download.DownloaderProgressPanel = downloaderProgressPanel.gameObject; + // Set up lines + line_panel = UICanvas.transform.Find("LinePanel").gameObject; + lines = UICanvas.transform.Find("LinePanel/Lines").gameObject; + line = UICanvas.transform.Find("LinePanel/Lines/Line").gameObject; + text_panel = UICanvas.transform.Find("TextPanel").gameObject; + text_base = UICanvas.transform.Find("TextPanel/TextHolder").gameObject; + + raw_image = UICanvas.transform.Find("RawImage").gameObject.GetComponent(); + label_text = UICanvas.transform.Find("LabelText").gameObject.GetComponent(); + mode_text = UICanvas.transform.Find("ModeLabel").gameObject.GetComponent(); + } + + void OnApplicationQuit() + { + DestroyAilia(); + ailia_camera.DestroyCamera(); + } + + void OnDestroy() + { + DestroyAilia(); + ailia_camera.DestroyCamera(); + } + } +} \ No newline at end of file diff --git a/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaImageCaptioningSample.cs.meta b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaImageCaptioningSample.cs.meta new file mode 100644 index 00000000..10cd384a --- /dev/null +++ b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/AiliaImageCaptioningSample.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2051cfa1f0fb6774bb4f5c15e8f51120 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/AXIP/AILIA-MODELS/ImageCaptioning/ImageCaptioningSample.unity b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/ImageCaptioningSample.unity new file mode 100644 index 00000000..dfe0d6fb --- /dev/null +++ b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/ImageCaptioningSample.unity @@ -0,0 +1,606 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!29 &1 +OcclusionCullingSettings: + m_ObjectHideFlags: 0 + serializedVersion: 2 + m_OcclusionBakeSettings: + smallestOccluder: 5 + smallestHole: 0.25 + backfaceThreshold: 100 + m_SceneGUID: 00000000000000000000000000000000 + m_OcclusionCullingData: {fileID: 0} +--- !u!104 &2 +RenderSettings: + m_ObjectHideFlags: 0 + serializedVersion: 9 + m_Fog: 0 + m_FogColor: {r: 0.5, g: 0.5, b: 0.5, a: 1} + m_FogMode: 3 + m_FogDensity: 0.01 + m_LinearFogStart: 0 + m_LinearFogEnd: 300 + m_AmbientSkyColor: {r: 0.212, g: 0.227, b: 0.259, a: 1} + m_AmbientEquatorColor: {r: 0.114, g: 0.125, b: 0.133, a: 1} + m_AmbientGroundColor: {r: 0.047, g: 0.043, b: 0.035, a: 1} + m_AmbientIntensity: 1 + m_AmbientMode: 0 + m_SubtractiveShadowColor: {r: 0.42, g: 0.478, b: 0.627, a: 1} + m_SkyboxMaterial: {fileID: 10304, guid: 0000000000000000f000000000000000, type: 0} + m_HaloStrength: 0.5 + m_FlareStrength: 1 + m_FlareFadeSpeed: 3 + m_HaloTexture: {fileID: 0} + m_SpotCookie: {fileID: 10001, guid: 0000000000000000e000000000000000, type: 0} + m_DefaultReflectionMode: 0 + m_DefaultReflectionResolution: 128 + m_ReflectionBounces: 1 + m_ReflectionIntensity: 1 + m_CustomReflection: {fileID: 0} + m_Sun: {fileID: 0} + m_IndirectSpecularColor: {r: 0.44657898, g: 0.4964133, b: 0.5748178, a: 1} + m_UseRadianceAmbientProbe: 0 +--- !u!157 &3 +LightmapSettings: + m_ObjectHideFlags: 0 + serializedVersion: 11 + m_GIWorkflowMode: 0 + m_GISettings: + serializedVersion: 2 + m_BounceScale: 1 + m_IndirectOutputScale: 1 + m_AlbedoBoost: 1 + m_EnvironmentLightingMode: 0 + m_EnableBakedLightmaps: 1 + m_EnableRealtimeLightmaps: 1 + m_LightmapEditorSettings: + serializedVersion: 12 + m_Resolution: 2 + m_BakeResolution: 40 + m_AtlasSize: 1024 + m_AO: 0 + m_AOMaxDistance: 1 + m_CompAOExponent: 1 + m_CompAOExponentDirect: 0 + m_ExtractAmbientOcclusion: 0 + m_Padding: 2 + m_LightmapParameters: {fileID: 0} + m_LightmapsBakeMode: 1 + m_TextureCompression: 1 + m_FinalGather: 0 + m_FinalGatherFiltering: 1 + m_FinalGatherRayCount: 256 + m_ReflectionCompression: 2 + m_MixedBakeMode: 1 + m_BakeBackend: 0 + m_PVRSampling: 1 + m_PVRDirectSampleCount: 32 + m_PVRSampleCount: 500 + m_PVRBounces: 2 + m_PVREnvironmentSampleCount: 500 + m_PVREnvironmentReferencePointCount: 2048 + m_PVRFilteringMode: 0 + m_PVRDenoiserTypeDirect: 0 + m_PVRDenoiserTypeIndirect: 0 + m_PVRDenoiserTypeAO: 0 + m_PVRFilterTypeDirect: 0 + m_PVRFilterTypeIndirect: 0 + m_PVRFilterTypeAO: 0 + m_PVREnvironmentMIS: 0 + m_PVRCulling: 1 + m_PVRFilteringGaussRadiusDirect: 1 + m_PVRFilteringGaussRadiusIndirect: 5 + m_PVRFilteringGaussRadiusAO: 2 + m_PVRFilteringAtrousPositionSigmaDirect: 0.5 + m_PVRFilteringAtrousPositionSigmaIndirect: 2 + m_PVRFilteringAtrousPositionSigmaAO: 1 + m_ExportTrainingData: 0 + m_TrainingDataDestination: TrainingData + m_LightProbeSampleCountMultiplier: 4 + m_LightingDataAsset: {fileID: 0} + m_UseShadowmask: 0 +--- !u!196 &4 +NavMeshSettings: + serializedVersion: 2 + m_ObjectHideFlags: 0 + m_BuildSettings: + serializedVersion: 2 + agentTypeID: 0 + agentRadius: 0.5 + agentHeight: 2 + agentSlope: 45 + agentClimb: 0.4 + ledgeDropHeight: 0 + maxJumpAcrossDistance: 0 + minRegionArea: 2 + manualCellSize: 0 + cellSize: 0.16666667 + manualTileSize: 0 + tileSize: 256 + accuratePlacement: 0 + debug: + m_Flags: 0 + m_NavMeshData: {fileID: 0} +--- !u!114 &66576881 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 114248270574527214, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + m_PrefabInstance: {fileID: 284098717} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!1 &134548559 stripped +GameObject: + m_CorrespondingSourceObject: {fileID: 1390452760564612, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + m_PrefabInstance: {fileID: 284098717} + m_PrefabAsset: {fileID: 0} +--- !u!1001 &284098717 +PrefabInstance: + m_ObjectHideFlags: 0 + serializedVersion: 2 + m_Modification: + m_TransformParent: {fileID: 0} + m_Modifications: + - target: {fileID: 223000755135152570, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_Camera + value: + objectReference: {fileID: 900571250} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_LocalPosition.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_LocalPosition.y + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_LocalPosition.z + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_LocalRotation.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_LocalRotation.y + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_LocalRotation.z + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_LocalRotation.w + value: 1 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_RootOrder + value: 4 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_AnchoredPosition.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_AnchoredPosition.y + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_SizeDelta.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_SizeDelta.y + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_AnchorMin.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_AnchorMin.y + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_AnchorMax.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_AnchorMax.y + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_Pivot.x + value: 0 + objectReference: {fileID: 0} + - target: {fileID: 224061873275276470, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + propertyPath: m_Pivot.y + value: 0 + objectReference: {fileID: 0} + m_RemovedComponents: [] + m_SourcePrefab: {fileID: 100100000, guid: b13fba7fbe84549eaad325ff3431cb6f, type: 3} +--- !u!1 &284098718 stripped +GameObject: + m_CorrespondingSourceObject: {fileID: 1829679373486840, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + m_PrefabInstance: {fileID: 284098717} + m_PrefabAsset: {fileID: 0} +--- !u!1 &437166881 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 437166882} + - component: {fileID: 437166883} + m_Layer: 0 + m_Name: ClassifierController + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!4 &437166882 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 437166881} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 481, y: 250, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 3 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!114 &437166883 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 437166881} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 28a1638e0d3db2246941ef9ef114937f, type: 3} + m_Name: + m_EditorClassIdentifier: + line_panel: {fileID: 1255283545} + lines: {fileID: 1779762464} + line: {fileID: 284098718} + text_panel: {fileID: 1966900788} + text_base: {fileID: 134548559} + ailiaModelType: 2 + UICanvas: {fileID: 1166538630} + gpu_mode: 1 + is_english: 0 + camera_id: 0 + raw_image: {fileID: 792245433} + mode_text: {fileID: 66576881} + label_text: {fileID: 1581251166} + resnet50model: resnet50.opt +--- !u!1 &489979542 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 489979544} + - component: {fileID: 489979543} + m_Layer: 0 + m_Name: Directional Light + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!108 &489979543 +Light: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 489979542} + m_Enabled: 1 + serializedVersion: 10 + m_Type: 1 + m_Shape: 0 + m_Color: {r: 1, g: 0.95686275, b: 0.8392157, a: 1} + m_Intensity: 1 + m_Range: 10 + m_SpotAngle: 30 + m_InnerSpotAngle: 21.80208 + m_CookieSize: 10 + m_Shadows: + m_Type: 2 + m_Resolution: -1 + m_CustomResolution: -1 + m_Strength: 1 + m_Bias: 0.05 + m_NormalBias: 0.4 + m_NearPlane: 0.2 + m_CullingMatrixOverride: + e00: 1 + e01: 0 + e02: 0 + e03: 0 + e10: 0 + e11: 1 + e12: 0 + e13: 0 + e20: 0 + e21: 0 + e22: 1 + e23: 0 + e30: 0 + e31: 0 + e32: 0 + e33: 1 + m_UseCullingMatrixOverride: 0 + m_Cookie: {fileID: 0} + m_DrawHalo: 0 + m_Flare: {fileID: 0} + m_RenderMode: 0 + m_CullingMask: + serializedVersion: 2 + m_Bits: 4294967295 + m_RenderingLayerMask: 1 + m_Lightmapping: 4 + m_LightShadowCasterMode: 0 + m_AreaSize: {x: 1, y: 1} + m_BounceIntensity: 1 + m_ColorTemperature: 6570 + m_UseColorTemperature: 0 + m_BoundingSphereOverride: {x: 0, y: 0, z: 0, w: 0} + m_UseBoundingSphereOverride: 0 + m_ShadowRadius: 0 + m_ShadowAngle: 0 +--- !u!4 &489979544 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 489979542} + m_LocalRotation: {x: 0.40821788, y: -0.23456968, z: 0.10938163, w: 0.8754261} + m_LocalPosition: {x: 0, y: 3, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 1 + m_LocalEulerAnglesHint: {x: 50, y: -30, z: 0} +--- !u!1 &663354113 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 663354116} + - component: {fileID: 663354115} + - component: {fileID: 663354114} + m_Layer: 0 + m_Name: EventSystem + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!114 &663354114 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 663354113} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 4f231c4fb786f3946a6b90b886c48677, type: 3} + m_Name: + m_EditorClassIdentifier: + m_HorizontalAxis: Horizontal + m_VerticalAxis: Vertical + m_SubmitButton: Submit + m_CancelButton: Cancel + m_InputActionsPerSecond: 10 + m_RepeatDelay: 0.5 + m_ForceModuleActive: 0 +--- !u!114 &663354115 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 663354113} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 76c392e42b5098c458856cdf6ecaaaa1, type: 3} + m_Name: + m_EditorClassIdentifier: + m_FirstSelected: {fileID: 0} + m_sendNavigationEvents: 1 + m_DragThreshold: 5 +--- !u!4 &663354116 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 663354113} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 2 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} +--- !u!114 &792245433 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 114335148288096466, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + m_PrefabInstance: {fileID: 284098717} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!1 &900571246 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 900571251} + - component: {fileID: 900571250} + - component: {fileID: 900571248} + - component: {fileID: 900571247} + m_Layer: 0 + m_Name: Main Camera + m_TagString: MainCamera + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!81 &900571247 +AudioListener: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 900571246} + m_Enabled: 1 +--- !u!124 &900571248 +Behaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 900571246} + m_Enabled: 1 +--- !u!20 &900571250 +Camera: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 900571246} + m_Enabled: 1 + serializedVersion: 2 + m_ClearFlags: 2 + m_BackGroundColor: {r: 0, g: 0, b: 0, a: 0} + m_projectionMatrixMode: 1 + m_GateFitMode: 2 + m_FOVAxisMode: 0 + m_SensorSize: {x: 36, y: 24} + m_LensShift: {x: 0, y: 0} + m_FocalLength: 50 + m_NormalizedViewPortRect: + serializedVersion: 2 + x: 0 + y: 0 + width: 1 + height: 1 + near clip plane: 0.3 + far clip plane: 1000 + field of view: 60 + orthographic: 0 + orthographic size: 5 + m_Depth: -1 + m_CullingMask: + serializedVersion: 2 + m_Bits: 4294967295 + m_RenderingPath: -1 + m_TargetTexture: {fileID: 0} + m_TargetDisplay: 0 + m_TargetEye: 3 + m_HDR: 0 + m_AllowMSAA: 1 + m_AllowDynamicResolution: 0 + m_ForceIntoRT: 0 + m_OcclusionCulling: 1 + m_StereoConvergence: 10 + m_StereoSeparation: 0.022 +--- !u!4 &900571251 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 900571246} + m_LocalRotation: {x: 0, y: 0, z: 1, w: 0} + m_LocalPosition: {x: 0, y: 0, z: -10} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 0} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 180} +--- !u!1 &1166538630 stripped +GameObject: + m_CorrespondingSourceObject: {fileID: 1716708668608742, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + m_PrefabInstance: {fileID: 284098717} + m_PrefabAsset: {fileID: 0} +--- !u!1 &1255283545 stripped +GameObject: + m_CorrespondingSourceObject: {fileID: 1018917012828270, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + m_PrefabInstance: {fileID: 284098717} + m_PrefabAsset: {fileID: 0} +--- !u!114 &1581251166 stripped +MonoBehaviour: + m_CorrespondingSourceObject: {fileID: 114356936467279116, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + m_PrefabInstance: {fileID: 284098717} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: +--- !u!1 &1779762464 stripped +GameObject: + m_CorrespondingSourceObject: {fileID: 1562200783443766, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + m_PrefabInstance: {fileID: 284098717} + m_PrefabAsset: {fileID: 0} +--- !u!1 &1966900788 stripped +GameObject: + m_CorrespondingSourceObject: {fileID: 1945808911574990, guid: b13fba7fbe84549eaad325ff3431cb6f, + type: 3} + m_PrefabInstance: {fileID: 284098717} + m_PrefabAsset: {fileID: 0} diff --git a/Assets/AXIP/AILIA-MODELS/ImageCaptioning/ImageCaptioningSample.unity.meta b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/ImageCaptioningSample.unity.meta new file mode 100644 index 00000000..351e7094 --- /dev/null +++ b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/ImageCaptioningSample.unity.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 963d1eb60f29c4043a5690bfae4088fb +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/AXIP/AILIA-MODELS/ImageCaptioning/LICENSE b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/LICENSE new file mode 100644 index 00000000..141a5790 --- /dev/null +++ b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Ruotian(RT) Luo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Assets/AXIP/AILIA-MODELS/ImageCaptioning/LICENSE.meta b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/LICENSE.meta new file mode 100644 index 00000000..4a24e114 --- /dev/null +++ b/Assets/AXIP/AILIA-MODELS/ImageCaptioning/LICENSE.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 538e3b4af8e1e7a4c879bb60ee7abbac +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/AXIP/AILIA-MODELS/common/Scripts/AiliaCamera.cs b/Assets/AXIP/AILIA-MODELS/common/Scripts/AiliaCamera.cs index 8912a351..b8a33953 100755 --- a/Assets/AXIP/AILIA-MODELS/common/Scripts/AiliaCamera.cs +++ b/Assets/AXIP/AILIA-MODELS/common/Scripts/AiliaCamera.cs @@ -21,6 +21,7 @@ public class AiliaCamera //WebCamera Instance private WebCamTexture webcamTexture = null; + private Texture2D webcamTexture2d = null; // Texture buffer Color32[] image = new Color32[0]; @@ -133,6 +134,22 @@ public Color32[] GetPixels32() return crop; } + public Texture2D GetTexture2D() + { + if (webcamTexture == null) + { + return null; + } + + if (webcamTexture2d == null) + { + webcamTexture2d = new Texture2D(webcamTexture.width, webcamTexture.height, TextureFormat.RGBA32, webcamTexture.mipmapCount, true); + } + + Graphics.CopyTexture(webcamTexture, webcamTexture2d); + return webcamTexture2d; + } + public int GetWidth() { if (webcamTexture.height > webcamTexture.width) diff --git a/Assets/AXIP/AILIA-MODELS/common/Scripts/AiliaDownload.cs b/Assets/AXIP/AILIA-MODELS/common/Scripts/AiliaDownload.cs index b49befb8..e6b9dc49 100755 --- a/Assets/AXIP/AILIA-MODELS/common/Scripts/AiliaDownload.cs +++ b/Assets/AXIP/AILIA-MODELS/common/Scripts/AiliaDownload.cs @@ -194,7 +194,7 @@ public IEnumerator DownloadWithProgressFromURL(List urlList, A // Error if (www.isHttpError || www.isNetworkError) { - Debug.LogError(www.error); + Debug.LogError($"Error fetching '{url}': {www.error}"); content += "" + www.error + "" + "\n"; if (ContentsText.cachedTextGenerator.lineCount > ContentLineCount) { diff --git a/Assets/AXIP/AILIA/Scripts/Models/AiliaModel.cs b/Assets/AXIP/AILIA/Scripts/Models/AiliaModel.cs index 834d2ec4..d93b5ead 100755 --- a/Assets/AXIP/AILIA/Scripts/Models/AiliaModel.cs +++ b/Assets/AXIP/AILIA/Scripts/Models/AiliaModel.cs @@ -513,5 +513,11 @@ public virtual void Close() ailia = IntPtr.Zero; } } + + //エラー詳細の取得 + public string GetErrorDetail() + { + return Marshal.PtrToStringAnsi(Ailia.ailiaGetErrorDetail(ailia)); + } } } \ No newline at end of file diff --git a/README.md b/README.md index 4eb5ab4f..23189835 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ We are now converting to C#. Please wait to complete conversion. | [illnet](/Assets/AXIP/AILIA-MODELS/ImageManipulation/) | [Document Rectification and Illumination Correction using a Patch-based CNN](https://github.com/xiaoyu258/DocProj) | Pytorch | 1.2.2 and later | | [colorization](/Assets/AXIP/AILIA-MODELS/ImageManipulation/) | [Colorful Image Colorization](https://github.com/richzhang/colorization) | Pytorch | 1.2.2 and later | +## Image captioning + +| Name | Detail | Exported From | Supported Ailia Version | +|:-----------|------------:|:------------:|:------------:| +| [image_captioning_pytorch](/Assets/AXIP/AILIA-MODELS/ImageCaptioning/)|[Image Captioning pytorch](https://github.com/ruotianluo/ImageCaptioning.pytorch) | Pytorch | 1.2.5 and later | + ## Object detection | Name | Detail | Exported From | Supported Ailia Version |