+init plugin unreal 5.0.3

PsyCV.uplugin

+{
+	"FileVersion": 3,
+	"Version": 1,
+	"VersionName": "1.0",
+	"FriendlyName": "PsyNNEngine",
+	"Description": "",
+	"Category": "AI",
+	"CreatedBy": "",
+	"CreatedByURL": "",
+	"DocsURL": "",
+	"MarketplaceURL": "",
+	"SupportURL": "",
+	"EngineVersion": "5.0.0",
+	"CanContainContent": true,
+	"Installed": true,
+	"Modules": [
+		{
+			"Name": "TextureProcessing",
+			"Type": "Runtime",
+			"LoadingPhase": "Default",
+			"PlatformAllowList": [
+				"Win64"
+			]
+		},
+		{
+			"Name": "PsyOpenCV",
+			"Type": "Runtime",
+			"LoadingPhase": "Default",
+			"PlatformAllowList": [
+				"Win64"
+			]
+		}
+	]
+}
\ No newline at end of file

Source/PsyOpenCV/Private/PsyOpenCV.cpp

+
+
+#include "PsyOpenCV.h"
+#include "Core.h"
+#include "Modules/ModuleManager.h"
+#include "Interfaces/IPluginManager.h"
+#include "HAL/PlatformProcess.h"
+
+#define LOCTEXT_NAMESPACE "FOpenCvUserModule"
+
+void FPsyOpenCVModule::StartupModule()
+{
+#if PLATFORM_WINDOWS
+	// This code will execute after your module is loaded into memory; the exact timing is specified in the .uplugin file per-module
+	const FString pluginDir = IPluginManager::Get().FindPlugin(TEXT("PsyCV"))->GetBaseDir();
+	const FString binDir = pluginDir / TEXT("Source/ThirdParty/opencv-4.4.0+contrib/bin");
+	const FString dLLPath = binDir / TEXT("opencv_world440.dll");
+
+	FPlatformProcess::PushDllDirectory(*binDir);
+	dllHandle = FPlatformProcess::GetDllHandle(*(dLLPath));
+	FPlatformProcess::PopDllDirectory(*binDir);
+#endif
+}
+
+void FPsyOpenCVModule::ShutdownModule()
+{
+	// This function may be called during shutdown to clean up your module.  For modules that support dynamic reloading,
+
+}
+
+#undef LOCTEXT_NAMESPACE
+	
+IMPLEMENT_MODULE(FPsyOpenCVModule, PsyOpenCV)

Source/PsyOpenCV/PsyOpenCV.build.cs

+using System.IO;
+using UnrealBuildTool;
+
+public class PsyOpenCV : ModuleRules
+{
+	private string ModulePath
+	{
+		get { return ModuleDirectory; }
+	}
+
+	private string ThirdPartyPath
+	{
+		get { return Path.GetFullPath(Path.Combine(ModulePath, "../ThirdParty/")); }
+	}
+
+	public PsyOpenCV(ReadOnlyTargetRules Target) : base(Target)
+	{
+		// set "false" to disable this OpenCV module
+        bool doUsePsyOpenCV = true;
+
+        PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
+		
+		PublicIncludePaths.AddRange(
+			new string[] {
+				// ... add public include paths required here ...
+			}
+			);
+				
+		
+		PrivateIncludePaths.AddRange(
+			new string[] {
+				// ... add other private include paths required here ...
+			}
+			);
+			
+		
+		PublicDependencyModuleNames.AddRange(
+			new string[]
+			{
+				"Core",
+				"CoreUObject",
+				"Projects",
+			}
+			);
+			
+		
+		PrivateDependencyModuleNames.AddRange(
+			new string[]
+			{
+				// ... add private dependencies that you statically link with here ...	
+			}
+			);
+		
+		
+		DynamicallyLoadedModuleNames.AddRange(
+			new string[]
+			{
+				// ... add any modules that your module loads dynamically here ...
+			}
+			);
+
+		if (doUsePsyOpenCV)
+        {
+			if (Target.Platform == UnrealTargetPlatform.Win64)
+			{
+				PublicIncludePaths.AddRange(new string[] { Path.Combine(ThirdPartyPath, "opencv-4.4.0+contrib", "include") });
+				PublicAdditionalLibraries.Add(Path.Combine(ThirdPartyPath, "opencv-4.4.0+contrib", "lib", "opencv_world440.lib"));
+				PublicDelayLoadDLLs.Add("opencv_world440.dll");
+				RuntimeDependencies.Add(/*"$(ProjectDir)/Binaries/Win64/opencv_world440.dll",*/ Path.Combine(ThirdPartyPath, "opencv-4.4.0+contrib", "bin", "opencv_world440.dll"));
+			}
+			
+			PublicDefinitions.Add("WITH_PSY_OPENCV=1");
+		}
+        else
+        {
+			PublicDefinitions.Add("WITH_PSY_OPENCV=0");
+		}
+	}
+}

Source/PsyOpenCV/PsyOpenCV_APL.xml

+<?xml version="1.0" encoding="utf-8"?>
+<root xmlns:android="http://schemas.android.com/apk/res/android">
+
+	<resourceCopies>
+		<log text="Copying lib*.so"/>
+		<!-- note: have to stage this since we linked it -->
+		<copyFile src="$S(AbsPluginDir)/../ThirdParty/opencv-4.5.4-android-sdk-native/libs/$S(Architecture)/libopencv_java4.so"
+					dst="$S(BuildDir)/libs/$S(Architecture)/libopencv_java4.so" />
+	</resourceCopies>
+
+</root>

Source/PsyOpenCV/Public/PsyOpenCV.h

+
+
+#pragma once
+
+#include "Modules/ModuleManager.h"
+
+class FPsyOpenCVModule : public IModuleInterface
+{
+public:
+	void* dllHandle;
+
+	/** IModuleInterface implementation */
+	virtual void StartupModule() override;
+	virtual void ShutdownModule() override;
+};

Source/TextureProcessing/Private/TextureProcessComponent.cpp

+
+
+#include "TextureProcessComponent.h"
+#include "Engine/TextureRenderTarget2D.h"
+#include "Async/ParallelFor.h"
+
+#if WITH_PSY_OPENCV
+#include "opencv2/core/core.hpp"
+#include <opencv2/imgproc.hpp>
+#endif
+
+// Sets default values for this component's properties
+UTextureProcessComponent::UTextureProcessComponent()
+{
+	// Set this component to be initialized when the game starts, and to be ticked every frame.  You can turn these features
+	// off to improve performance if you don't need them.
+	PrimaryComponentTick.bCanEverTick = false;
+
+	// ...
+}
+
+void UTextureProcessComponent::BeginPlay()
+{
+	renderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8_SRGB;
+
+	Super::BeginPlay();
+
+	outputImageBuffer.Init(0, destinationWidth * destinationHeight * 3);
+}
+
+void UTextureProcessComponent::resize(UTexture* inputTexture, TArray<uint8>& outputHxWxBGR, FVector2D& uvScalingFactor, const bool doFlipImage, const EImageRotation doRotateImage)
+{
+	// execute resizing by Material
+	UTextureRenderTarget2D* renderTarget = Super::resize(inputTexture, uvScalingFactor, doFlipImage, doRotateImage);
+
+	// get raw pixel data from render target
+	getPixelData(renderTarget, outputHxWxBGR);
+}
+
+void UTextureProcessComponent::getPixelData(UTextureRenderTarget2D* renderTarget, TArray<uint8>& outputHxWxBGR)
+{
+	// get pixel data
+	TArray<FColor> textureRawData;
+	FRenderTarget* texResource = renderTarget->GameThread_GetRenderTargetResource();
+	texResource->ReadPixels(textureRawData);
+
+	// convert BGRA to BGR
+#if PLATFORM_ANDROID
+	const int numOfPixels = textureRawData.Num();
+	TArray<uint8>& tempArray = outputImageBuffer;
+	const int width = destinationWidth;
+	const int height = destinationHeight;
+	ParallelFor(numOfPixels, [&tempArray, &textureRawData, &width, &height](int32 i)
+		{
+			// The result of "ReadPixels" is flipped vertically in Android
+			const int srcRow = i / width;
+			const int srcCol = i - srcRow * width;
+			const int dstRow = height - srcRow - 1;
+			const int dstCol = srcCol;
+			const int dstIndex = dstRow * width + dstCol;
+
+			const int outOffset = dstIndex * 3;
+			tempArray[outOffset    ] = textureRawData[i].B;
+			tempArray[outOffset + 1] = textureRawData[i].G;
+			tempArray[outOffset + 2] = textureRawData[i].R;
+		});
+#else //PLATFORM_ANDROID
+#if WITH_PSY_OPENCV
+	cv::Mat matFromRawData = cv::Mat(cv::Size(destinationWidth, destinationHeight), CV_8UC4, textureRawData.GetData());
+	cv::Mat ret =            cv::Mat(cv::Size(destinationWidth, destinationHeight), CV_8UC3, outputImageBuffer.GetData());
+	cv::cvtColor(matFromRawData, ret, cv::COLOR_BGRA2BGR);
+#else //WITH_PSY_OPENCV
+	const int numOfPixels = textureRawData.Num();
+	TArray<uint8>& tempArray = outputImageBuffer;
+	ParallelFor(numOfPixels, [&tempArray, &textureRawData](int32 i)
+		{
+			const int outOffset = i * 3;
+			tempArray[outOffset    ] = textureRawData[i].B;
+			tempArray[outOffset + 1] = textureRawData[i].G;
+			tempArray[outOffset + 2] = textureRawData[i].R;
+		});
+#endif //WITH_PSY_OPENCV
+#endif //PLATFORM_ANDROID
+
+	// return value
+	outputHxWxBGR = outputImageBuffer;
+}
+
+void UTextureProcessComponent::affineTransform(UTexture* inputTexture, TArray<uint8>& outputHxWxBGR, const FAffineMat2D& inverseNormalizedAffineMat, const bool doFlipImage, const EImageRotation doRotateImage)
+{
+	// execute affine transform by Material
+	UTextureRenderTarget2D* renderTarget = renderAffineTransformedFrame(inputTexture, inverseNormalizedAffineMat, doFlipImage, doRotateImage);
+
+	// get raw pixel data from render target
+	getPixelData(renderTarget, outputHxWxBGR);
+}
+
+

Source/TextureProcessing/Private/TextureProcessComponentBase.cpp

+
+
+#include "TextureProcessComponentBase.h"
+#include "Kismet/KismetRenderingLibrary.h"
+#include "Kismet/KismetMaterialLibrary.h"
+#include "TextureProcessFunctionLibrary.h"
+
+// Sets default values for this component's properties
+UTextureProcessComponentBase::UTextureProcessComponentBase()
+{
+	// Set this component to be initialized when the game starts, and to be ticked every frame.  You can turn these features
+	// off to improve performance if you don't need them.
+	PrimaryComponentTick.bCanEverTick = false;
+
+	// ...
+}
+
+void UTextureProcessComponentBase::BeginPlay()
+{
+	Super::BeginPlay();
+
+	// render target
+	renderTargetAffineWarp = UKismetRenderingLibrary::CreateRenderTarget2D(this, destinationWidth, destinationHeight, renderTargetFormat);
+
+	// materials for affine warp
+	UMaterialInterface* parent_external = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(), nullptr, TEXT("/NNEngine/TextureProcessing/AffineTransformMat")));
+	UMaterialInterface* parent_color    = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(), nullptr, TEXT("/NNEngine/TextureProcessing/AffineTransformMat_color")));
+	materialAffineWarp_external = UKismetMaterialLibrary::CreateDynamicMaterialInstance(this, parent_external);
+	materialAffineWarp_color    = UKismetMaterialLibrary::CreateDynamicMaterialInstance(this, parent_color);
+}
+
+void UTextureProcessComponentBase::calcAspectRatio(UTexture* inputTexture, FVector2D& uvScalingFactor)
+{
+	const float inputWidth = inputTexture->GetSurfaceWidth() / destinationWidth;
+	const float inputHeight = inputTexture->GetSurfaceHeight() / destinationHeight;
+	if (inputWidth > inputHeight)
+	{
+		// Landscape
+		uvScalingFactor.X = 1.0f;
+		uvScalingFactor.Y = inputWidth / inputHeight; // > 1.0f
+	}
+	else
+	{
+		// Portrait
+		uvScalingFactor.X = inputHeight / inputWidth; // > 1.0f
+		uvScalingFactor.Y = 1.0f;
+	}
+}
+
+UTextureRenderTarget2D* UTextureProcessComponentBase::resize(UTexture* inputTexture, FVector2D& uvScalingFactor, const bool doFlipImage, const EImageRotation doRotateImage)
+{
+	calcAspectRatio(inputTexture, uvScalingFactor);
+
+	FAffineMat2D inverseNormalizedAffineMat;
+	inverseNormalizedAffineMat.a = uvScalingFactor.X; inverseNormalizedAffineMat.b = 0.0f;              inverseNormalizedAffineMat.tx = 0.0f;
+	inverseNormalizedAffineMat.c = 0.0f;              inverseNormalizedAffineMat.d = uvScalingFactor.Y; inverseNormalizedAffineMat.ty = 0.0f;
+
+	// execute resizing by Material, implemented by BP
+	return renderAffineTransformedFrame(inputTexture, inverseNormalizedAffineMat, doFlipImage, doRotateImage);
+}
+
+UTextureRenderTarget2D* UTextureProcessComponentBase::renderAffineTransformedFrame(UTexture* inputTexture, const FAffineMat2D& inverseNormalizedAffineMat, const bool doFlipImage, const EImageRotation doRotateImage)
+{
+	// assign material
+	UMaterialInstanceDynamic* material = nullptr;
+	if (UTextureProcessFunctionLibrary::isExternal(inputTexture))
+	{
+		material = materialAffineWarp_external;
+	}
+	else
+	{
+		material = materialAffineWarp_color;
+	}
+
+	// set parameters
+	material->SetVectorParameterValue("abcd", FLinearColor(inverseNormalizedAffineMat.a, inverseNormalizedAffineMat.b, inverseNormalizedAffineMat.c, inverseNormalizedAffineMat.d));
+	material->SetVectorParameterValue("t"   , FLinearColor(inverseNormalizedAffineMat.tx, inverseNormalizedAffineMat.ty, 0.0f));
+	material->SetScalarParameterValue("Rotation", (float)doRotateImage);
+	if (doFlipImage)
+	{
+		material->SetScalarParameterValue("Flip", -1.0f);
+	}
+	else
+	{
+		material->SetScalarParameterValue("Flip", 1.0f);
+	}
+	material->SetTextureParameterValue("Texture", inputTexture);
+
+	// render
+	UKismetRenderingLibrary::DrawMaterialToRenderTarget(this, renderTargetAffineWarp, material);
+	return renderTargetAffineWarp;
+}

Source/TextureProcessing/Private/TextureProcessFloatComponent.cpp

+
+
+#include "TextureProcessFloatComponent.h"
+#include "Engine/TextureRenderTarget2D.h"
+#include "Async/ParallelFor.h"
+
+#if WITH_PSY_OPENCV
+#include "opencv2/core/core.hpp"
+#include <opencv2/imgproc.hpp>
+#endif
+
+// Sets default values for this component's properties
+UTextureProcessFloatComponent::UTextureProcessFloatComponent()
+{
+	// Set this component to be initialized when the game starts, and to be ticked every frame.  You can turn these features
+	// off to improve performance if you don't need them.
+	PrimaryComponentTick.bCanEverTick = false;
+
+	// ...
+}
+
+void UTextureProcessFloatComponent::BeginPlay()
+{
+	renderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA32f;
+
+	Super::BeginPlay();
+
+	outputImageBuffer.Init(0, destinationWidth * destinationHeight * 3);
+}
+
+void UTextureProcessFloatComponent::resize(UTexture* inputTexture, TArray<float>& output, FVector2D& uvScalingFactor, const bool doFlipImage, const EImageRotation doRotateImage, const bool toCHW)
+{
+	// execute resizing by Material
+	UTextureRenderTarget2D* renderTarget = Super::resize(inputTexture, uvScalingFactor, doFlipImage, doRotateImage);
+
+	// get raw pixel data from render target
+	getPixelData(renderTarget, output, toCHW);
+}
+
+void UTextureProcessFloatComponent::getPixelData(UTextureRenderTarget2D* renderTarget, TArray<float>& output, const bool toCHW)
+{
+	// get pixel data
+	TArray<FLinearColor> textureRawData;
+	FRenderTarget* texResource = renderTarget->GameThread_GetRenderTargetResource();
+	texResource->ReadLinearColorPixels(textureRawData);
+
+	if (toCHW)
+	{
+		const int numOfPixels = textureRawData.Num();
+		const int offset1 = numOfPixels;
+		const int offset2 = numOfPixels + numOfPixels;
+		TArray<float>& tempArray = outputImageBuffer;
+		ParallelFor(numOfPixels, [&tempArray, &textureRawData, offset1, offset2](int32 i)
+			{
+				tempArray[          i] = textureRawData[i].R;
+				tempArray[offset1 + i] = textureRawData[i].G;
+				tempArray[offset2 + i] = textureRawData[i].B;
+			});
+	}
+	else // H x W x C
+	{
+#if WITH_PSY_OPENCV
+		// convert RGBA to RGB
+		cv::Mat matFromRawData = cv::Mat(cv::Size(destinationWidth, destinationHeight), CV_32FC4, textureRawData.GetData());
+		cv::Mat ret            = cv::Mat(cv::Size(destinationWidth, destinationHeight), CV_32FC3, outputImageBuffer.GetData());
+		cv::cvtColor(matFromRawData, ret, cv::COLOR_RGBA2RGB);
+#else
+		const int numOfPixels = textureRawData.Num();
+		TArray<float>& tempArray = outputImageBuffer;
+		ParallelFor(numOfPixels, [&tempArray, &textureRawData](int32 i)
+			{
+				const int outOffset = i * 3;
+				tempArray[outOffset    ] = textureRawData[i].R;
+				tempArray[outOffset + 1] = textureRawData[i].G;
+				tempArray[outOffset + 2] = textureRawData[i].B;
+			});
+#endif
+	}
+
+	// return value
+	output = outputImageBuffer;
+}
+
+void UTextureProcessFloatComponent::affineTransform(UTexture* inputTexture, TArray<float>& outputRawImage, const FAffineMat2D& inverseNormalizedAffineMat, const bool doFlipImage, const EImageRotation doRotateImage, const bool HWC_to_CHW)
+{
+	// execute affine transform by Material
+	UTextureRenderTarget2D* renderTarget = renderAffineTransformedFrame(inputTexture, inverseNormalizedAffineMat, doFlipImage, doRotateImage);
+
+	// get raw pixel data from render target
+	getPixelData(renderTarget, outputRawImage, HWC_to_CHW);
+}
+