feat: Add asset loader, renderer, and OBS utils

This commit adds new files and functionality related to asset loading, rendering, and OBS utilities. It includes the following changes:

- Added `asset-loader.h` for loading assets using Assimp library.
- Added `obs-utils.h` for OBS utility functions.
- Added `asset-render.h` for rendering 3D assets using augmented filter data.
- Updated `plugin-support.h` and `plugin-main.c` to register the augmented filter source.
- Updated `buildspec.json` with new plugin information.
- Added `augmented-filter-data.h` for storing base data needed for ORT filters.
- Updated `plugin-support.c.in` with the plugin name and version.
- Added `augmented-filter.c` with the implementation of the augmented filter source.
- Updated `CMakeLists.txt` to include new source files and link necessary libraries.

These changes introduce new functionality to the project, enabling asset loading, rendering, and OBS integration for the augmented filter plugin.

```

CMakeLists.txt

@@ -34,6 +34,28 @@ if(ENABLE_QT)
                AUTORCC ON)
 endif()
 
-target_sources(${CMAKE_PROJECT_NAME} PRIVATE src/plugin-main.c)
+include(cmake/BuildAssimp.cmake)
+target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE assimp)
+
+set(USE_SYSTEM_OPENCV
+    OFF
+    CACHE STRING "Use system OpenCV")
+if(USE_SYSTEM_OPENCV)
+  if(OS_LINUX)
+    find_package(OpenCV REQUIRED COMPONENTS core imgproc)
+    target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE "${OpenCV_LIBRARIES}")
+    target_include_directories(${CMAKE_PROJECT_NAME} SYSTEM PUBLIC "${OpenCV_INCLUDE_DIRS}")
+  else()
+    message(FATAL_ERROR "System OpenCV is only supported on Linux!")
+  endif()
+else()
+  include(cmake/FetchOpenCV.cmake)
+  target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE OpenCV)
+endif()
+
+target_sources(
+  ${CMAKE_PROJECT_NAME}
+  PRIVATE src/plugin-main.c src/augmented-filter.c src/augmented-filter.cpp src/asset-utils/asset-loader.cpp
+          src/asset-utils/asset-render.cpp src/obs-utils/obs-utils.cpp)
 
 set_target_properties_plugin(${CMAKE_PROJECT_NAME} PROPERTIES OUTPUT_NAME ${_name})

buildspec.json

@@ -33,15 +33,15 @@
     },
     "platformConfig": {
         "macos": {
-            "bundleId": "com.example.obs-plugintemplate"
+            "bundleId": "com.royshilkrot.obs-augmented"
         }
     },
-    "name": "obs-plugintemplate",
-    "displayName": "OBS Plugin Template",
-    "version": "1.0.0",
-    "author": "Your Name Here",
-    "website": "https://example.com",
-    "email": "me@example.com",
+    "name": "obs-augmented",
+    "displayName": "OBS Augmented",
+    "version": "0.0.1",
+    "author": "Roy Shilkrot",
+    "website": "https://github.com/occ-ai/obs-augmented",
+    "email": "roy.shil@gmail.com",
     "uuids": {
         "windowsApp": "00000000-0000-0000-0000-000000000000"
     }

cmake/BuildAssimp.cmake

@@ -0,0 +1,50 @@
+include(ExternalProject)
+
+set(ASSIMP_VERSION 5.4.1)
+set(ASSIMP_URL https://github.com/assimp/assimp/archive/refs/tags/v${ASSIMP_VERSION}.tar.gz)
+if(WIN32)
+  set(ASSIMP_SHARE_LIB_NAME assimp-vc143-mt)
+else()
+  set(ASSIMP_SHARE_LIB_NAME assimp)
+endif()
+
+ExternalProject_Add(
+  assimp-build
+  URL ${ASSIMP_URL} DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+  CMAKE_GENERATOR ${CMAKE_GENERATOR}
+  CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
+             -DBUILD_SHARED_LIBS=ON
+             -DASSIMP_BUILD_ASSIMP_TOOLS=OFF
+             -DASSIMP_BUILD_TESTS=OFF
+             -DASSIMP_BUILD_SAMPLES=OFF
+             -DASSIMP_BUILD_ALL_IMPORTERS_BY_DEFAULT=ON
+             -DASSIMP_BUILD_ALL_EXPORTERS_BY_DEFAULT=OFF
+             -DASSIMP_INSTALL_PDB=OFF
+             -DASSIMP_NO_EXPORT=ON)
+
+ExternalProject_Get_Property(assimp-build INSTALL_DIR)
+
+add_library(assimp::assimp SHARED IMPORTED)
+add_dependencies(assimp::assimp assimp-build)
+set_target_properties(
+  assimp::assimp
+  PROPERTIES IMPORTED_LOCATION
+             ${INSTALL_DIR}/bin/${CMAKE_SHARED_LIBRARY_PREFIX}${ASSIMP_SHARE_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+if(WIN32)
+  set_target_properties(
+    assimp::assimp
+    PROPERTIES IMPORTED_IMPLIB
+               ${INSTALL_DIR}/lib/${CMAKE_IMPORT_LIBRARY_PREFIX}${ASSIMP_SHARE_LIB_NAME}${CMAKE_IMPORT_LIBRARY_SUFFIX})
+endif()
+target_include_directories(assimp::assimp INTERFACE ${INSTALL_DIR}/include)
+
+add_library(assimp INTERFACE)
+add_dependencies(assimp assimp-build)
+target_link_libraries(assimp INTERFACE assimp::assimp)
+target_include_directories(assimp INTERFACE ${INSTALL_DIR}/include)
+
+if(WIN32)
+  # install the DLL to the plugin output directory
+  install(FILES ${INSTALL_DIR}/bin/${CMAKE_SHARED_LIBRARY_PREFIX}${ASSIMP_SHARE_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
+          DESTINATION ${CMAKE_SOURCE_DIR}/release/${CMAKE_BUILD_TYPE}/obs-plugins/64bit)
+endif()

cmake/FetchOpenCV.cmake

@@ -0,0 +1,81 @@
+include(FetchContent)
+
+set(CUSTOM_OPENCV_URL
+    ""
+    CACHE STRING "URL of a downloaded OpenCV static library tarball")
+
+set(CUSTOM_OPENCV_HASH
+    ""
+    CACHE STRING "Hash of a downloaded OpenCV staitc library tarball")
+
+if(CUSTOM_OPENCV_URL STREQUAL "")
+  set(USE_PREDEFINED_OPENCV ON)
+else()
+  if(CUSTOM_OPENCV_HASH STREQUAL "")
+    message(FATAL_ERROR "Both of CUSTOM_OPENCV_URL and CUSTOM_OPENCV_HASH must be present!")
+  else()
+    set(USE_PREDEFINED_OPENCV OFF)
+  endif()
+endif()
+
+if(USE_PREDEFINED_OPENCV)
+  set(OpenCV_VERSION "v4.9.0-1")
+  set(OpenCV_BASEURL "https://github.com/obs-ai/obs-backgroundremoval-dep-opencv/releases/download/${OpenCV_VERSION}")
+
+  if(${CMAKE_BUILD_TYPE} STREQUAL Release OR ${CMAKE_BUILD_TYPE} STREQUAL RelWithDebInfo)
+    set(OpenCV_BUILD_TYPE Release)
+  else()
+    set(OpenCV_BUILD_TYPE Debug)
+  endif()
+
+  if(APPLE)
+    if(OpenCV_BUILD_TYPE STREQUAL Debug)
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-macos-${OpenCV_VERSION}-Debug.tar.gz")
+      set(OpenCV_HASH SHA256=BE85C8224F71C52162955BEE4EC9FFBE41CBED636D7989843CA75AD42657B121)
+    else()
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-macos-${OpenCV_VERSION}-Release.tar.gz")
+      set(OpenCV_HASH SHA256=5DB4FCFBD8C7CDBA136657B4D149821A670DF9A7C71120F5A4D34FA35A58D07B)
+    endif()
+  elseif(MSVC)
+    if(OpenCV_BUILD_TYPE STREQUAL Debug)
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-windows-${OpenCV_VERSION}-Debug.zip")
+      set(OpenCV_HASH SHA256=0A1BBC898DCE5F193427586DA84D7A34BBB783127957633236344E9CCD61B9CE)
+    else()
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-windows-${OpenCV_VERSION}-Release.zip")
+      set(OpenCV_HASH SHA256=56A5E042F490B8390B1C1819B2B48C858F10CD64E613BABBF11925A57269C3FA)
+    endif()
+  else()
+    if(OpenCV_BUILD_TYPE STREQUAL Debug)
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-linux-${OpenCV_VERSION}-Debug.tar.gz")
+      set(OpenCV_HASH SHA256=840A7D80B661CFF7B7300272A2A2992D539672ECECA01836B85F68BD8CAF07F5)
+    else()
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-linux-${OpenCV_VERSION}-Release.tar.gz")
+      set(OpenCV_HASH SHA256=73652C2155B477B5FD95FCD8EA7CE35D313543ECE17BDFA3A2B217A0239D74C6)
+    endif()
+  endif()
+else()
+  set(OpenCV_URL "${CUSTOM_OPENCV_URL}")
+  set(OpenCV_HASH "${CUSTOM_OPENCV_HASH}")
+endif()
+
+FetchContent_Declare(
+  opencv
+  URL ${OpenCV_URL}
+  URL_HASH ${OpenCV_HASH})
+FetchContent_MakeAvailable(opencv)
+
+add_library(OpenCV INTERFACE)
+if(MSVC)
+  target_link_libraries(
+    OpenCV
+    INTERFACE ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_imgproc490.lib
+              ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_core490.lib
+              ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_video490.lib
+              ${opencv_SOURCE_DIR}/x64/vc17/staticlib/zlib.lib)
+  target_include_directories(OpenCV SYSTEM INTERFACE ${opencv_SOURCE_DIR}/include)
+else()
+  target_link_libraries(
+    OpenCV INTERFACE ${opencv_SOURCE_DIR}/lib/libopencv_imgproc.a ${opencv_SOURCE_DIR}/lib/libopencv_core.a
+                     ${opencv_SOURCE_DIR}/lib/libopencv_video.a ${opencv_SOURCE_DIR}/lib/opencv4/3rdparty/libzlib.a)
+  target_include_directories(OpenCV SYSTEM INTERFACE ${opencv_SOURCE_DIR}/include/opencv4)
+endif()

src/asset-utils/asset-loader.cpp

@@ -0,0 +1,51 @@
+
+#include <assimp/cimport.h>
+#include <assimp/postprocess.h>
+#include <assimp/scene.h>
+
+#include <obs.h>
+
+#include "plugin-support.h"
+
+const aiScene *load_asset(const char *path)
+{
+	// Load asset from path using Assimp
+	const aiScene *scene =
+		aiImportFile(path, aiProcessPreset_TargetRealtime_MaxQuality);
+
+	if (!scene) {
+		obs_log(LOG_ERROR, "Failed to load asset: %s\n",
+			aiGetErrorString());
+		return nullptr;
+	}
+
+	// print asset info
+	obs_log(LOG_INFO, "Asset: %s", scene->mName.C_Str());
+	obs_log(LOG_INFO, "Number of meshes: %d", scene->mNumMeshes);
+	obs_log(LOG_INFO, "Number of materials: %d", scene->mNumMaterials);
+	obs_log(LOG_INFO, "Number of textures: %d", scene->mNumTextures);
+
+	// print mesh info
+	for (unsigned int i = 0; i < scene->mNumMeshes; i++) {
+		aiMesh *mesh = scene->mMeshes[i];
+		obs_log(LOG_INFO, "Mesh %d: %s", i, mesh->mName.C_Str());
+		obs_log(LOG_INFO, "Number of vertices: %d", mesh->mNumVertices);
+		obs_log(LOG_INFO, "Number of faces: %d", mesh->mNumFaces);
+	}
+
+	// print material info
+	for (unsigned int i = 0; i < scene->mNumMaterials; i++) {
+		aiMaterial *material = scene->mMaterials[i];
+		obs_log(LOG_INFO, "Material %d: %s", i,
+			material->GetName().C_Str());
+	}
+
+	// print texture info
+	for (unsigned int i = 0; i < scene->mNumTextures; i++) {
+		aiTexture *texture = scene->mTextures[i];
+		obs_log(LOG_INFO, "Texture %d: %s", i,
+			texture->mFilename.C_Str());
+	}
+
+	return scene;
+}

src/asset-utils/asset-loader.h

@@ -0,0 +1,8 @@
+#ifndef ASSET_LOADER_H
+#define ASSET_LOADER_H
+
+#include <assimp/scene.h>
+
+const aiScene *load_asset(const char *path);
+
+#endif /* ASSET_LOADER_H */

src/asset-utils/asset-render.cpp

@@ -0,0 +1,97 @@
+
+#include "asset-render.h"
+#include "augmented-filter-data.h"
+
+#include <obs.h>
+#include <graphics/graphics.h>
+
+#include <assimp/scene.h>
+
+bool render_asset_3d(augmented_filter_data *afd, const aiScene *scene)
+{
+	obs_source_t *target = obs_filter_get_target(afd->source);
+	if (!target) {
+		return false;
+	}
+	uint32_t width = obs_source_get_base_width(target);
+	uint32_t height = obs_source_get_base_height(target);
+	if (width == 0 || height == 0) {
+		return false;
+	}
+	gs_texrender_reset(afd->texrender);
+	gs_viewport_push();
+	gs_matrix_push();
+	gs_blend_state_push();
+	gs_blend_function(GS_BLEND_ONE, GS_BLEND_ZERO);
+	if (!gs_texrender_begin(afd->texrender, width, height)) {
+		return false;
+	}
+	struct vec4 background;
+	vec4_zero(&background);
+	gs_clear(GS_CLEAR_COLOR, &background, 0.0f, 0);
+
+	gs_perspective(120.0f, (float)width / (float)height,
+		       1.0f / (float)(1 << 22), (float)(1 << 22));
+
+	gs_vb_data *vb = gs_vbdata_create();
+	vb->points =
+		(vec3 *)bmalloc(sizeof(vec3) * scene->mMeshes[0]->mNumVertices);
+	vb->colors = (uint32_t *)bmalloc(sizeof(uint32_t) *
+					 scene->mMeshes[0]->mNumVertices);
+	vb->num = scene->mMeshes[0]->mNumVertices;
+	for (unsigned int i = 0; i < scene->mMeshes[0]->mNumVertices; i++) {
+		vb->points[i].x = scene->mMeshes[0]->mVertices[i].x;
+		vb->points[i].y = scene->mMeshes[0]->mVertices[i].y;
+		vb->points[i].z = scene->mMeshes[0]->mVertices[i].z;
+		vb->colors[i] = 0xFFFFFFFF;
+	}
+	gs_vertbuffer_t *vbo = gs_vertexbuffer_create(vb, GS_DYNAMIC);
+	gs_vertexbuffer_flush(vbo);
+	gs_load_vertexbuffer(vbo);
+	gs_load_indexbuffer(NULL);
+
+	gs_draw(GS_TRIS, 0, 0);
+
+	gs_vertexbuffer_destroy(vbo);
+
+	gs_matrix_identity();
+
+	gs_texrender_end(afd->texrender);
+	gs_blend_state_pop();
+	gs_matrix_pop();
+	gs_viewport_pop();
+
+	return true;
+}
+
+/*
+void effect_3d_draw_frame(struct effect_3d *context, uint32_t w, uint32_t h)
+{
+	const enum gs_color_space current_space = gs_get_color_space();
+	float multiplier;
+	const char *technique = get_tech_name_and_multiplier(
+		current_space, context->space, &multiplier);
+
+	gs_effect_t *effect = obs_get_base_effect(OBS_EFFECT_DEFAULT);
+	gs_texture_t *tex = gs_texrender_get_texture(context->render);
+	if (!tex)
+		return;
+
+	gs_blend_state_push();
+	gs_blend_function(GS_BLEND_ONE, GS_BLEND_INVSRCALPHA);
+
+	const bool previous = gs_framebuffer_srgb_enabled();
+	gs_enable_framebuffer_srgb(true);
+
+	gs_effect_set_texture_srgb(gs_effect_get_param_by_name(effect, "image"),
+				   tex);
+	gs_effect_set_float(gs_effect_get_param_by_name(effect, "multiplier"),
+			    multiplier);
+
+	while (gs_effect_loop(effect, technique))
+		gs_draw_sprite(tex, 0, w, h);
+
+	gs_enable_framebuffer_srgb(previous);
+	gs_blend_state_pop();
+}
+*/

src/asset-utils/asset-render.h

@@ -0,0 +1,8 @@
+#ifndef ASSET_RENDER_H
+#define ASSET_RENDER_H
+
+#include "augmented-filter-data.h"
+
+bool render_asset_3d(augmented_filter_data *afd, const aiScene *scene);
+
+#endif /* ASSET_RENDER_H */

src/augmented-filter-data.h

@@ -0,0 +1,38 @@
+#ifndef FILTERDATA_H
+#define FILTERDATA_H
+
+#include <obs.h>
+
+#include <opencv2/core.hpp>
+
+#include <assimp/scene.h>
+
+#include <mutex>
+#include <string>
+
+/**
+  * @brief The filter_data struct
+  *
+  * This struct is used to store the base data needed for ORT filters.
+  *
+*/
+struct augmented_filter_data {
+	obs_source_t *source;
+	gs_texrender_t *texrender;
+	gs_stagesurf_t *stagesurface;
+
+	const aiScene *asset;
+
+	cv::Mat inputBGRA;
+	cv::Mat outputPreviewBGRA;
+	cv::Mat outputMask;
+
+	bool isDisabled;
+	bool preview;
+
+	std::mutex inputBGRALock;
+	std::mutex outputLock;
+	std::mutex modelMutex;
+};
+
+#endif /* FILTERDATA_H */