/* -LICENSE-START- ** Copyright (c) 2012 Blackmagic Design ** ** Permission is hereby granted, free of charge, to any person or organization ** obtaining a copy of the software and accompanying documentation (the ** "Software") to use, reproduce, display, distribute, sub-license, execute, ** and transmit the Software, and to prepare derivative works of the Software, ** and to permit third-parties to whom the Software is furnished to do so, in ** accordance with: ** ** (1) if the Software is obtained from Blackmagic Design, the End User License ** Agreement for the Software Development Kit ("EULA") available at ** https://www.blackmagicdesign.com/EULA/DeckLinkSDK; or ** ** (2) if the Software is obtained from any third party, such licensing terms ** as notified by that third party, ** ** and all subject to the following: ** ** (3) the copyright notices in the Software and this entire statement, ** including the above license grant, this restriction and the following ** disclaimer, must be included in all copies of the Software, in whole or in ** part, and all derivative works of the Software, unless such copies or ** derivative works are solely in the form of machine-executable object code ** generated by a source language processor. ** ** (4) THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ** OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT ** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE ** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ** DEALINGS IN THE SOFTWARE. ** ** A copy of the Software is available free of charge at ** https://www.blackmagicdesign.com/desktopvideo_sdk under the EULA. ** ** -LICENSE-END- */ #include "ControlServer.h" #include "OpenGLComposite.h" #include "GLExtensions.h" #include "OscServer.h" #include #include #include #include #include #include #include #include #include #include DEFINE_GUID(IID_PinnedMemoryAllocator, 0xddf921a6, 0x279d, 0x4dcd, 0x86, 0x26, 0x75, 0x7f, 0x58, 0xa8, 0xc4, 0x35); namespace { constexpr GLuint kDecodedVideoTextureUnit = 1; constexpr GLuint kSourceHistoryTextureUnitBase = 2; constexpr GLuint kPackedVideoTextureUnit = 2; constexpr GLuint kGlobalParamsBindingPoint = 0; const char* kVertexShaderSource = "#version 430 core\n" "out vec2 vTexCoord;\n" "void main()\n" "{\n" " vec2 positions[3] = vec2[3](vec2(-1.0, -1.0), vec2(3.0, -1.0), vec2(-1.0, 3.0));\n" " vec2 texCoords[3] = vec2[3](vec2(0.0, 0.0), vec2(2.0, 0.0), vec2(0.0, 2.0));\n" " gl_Position = vec4(positions[gl_VertexID], 0.0, 1.0);\n" " vTexCoord = texCoords[gl_VertexID];\n" "}\n"; const char* kDecodeFragmentShaderSource = "#version 430 core\n" "layout(binding = 2) uniform sampler2D uPackedVideoInput;\n" "uniform vec2 uPackedVideoResolution;\n" "uniform vec2 uDecodedVideoResolution;\n" "in vec2 vTexCoord;\n" "layout(location = 0) out vec4 fragColor;\n" "vec4 rec709YCbCr2rgba(float Y, float Cb, float Cr, float a)\n" "{\n" " Y = (Y * 256.0 - 16.0) / 219.0;\n" " Cb = (Cb * 256.0 - 16.0) / 224.0 - 0.5;\n" " Cr = (Cr * 256.0 - 16.0) / 224.0 - 0.5;\n" " return vec4(Y + 1.5748 * Cr, Y - 0.1873 * Cb - 0.4681 * Cr, Y + 1.8556 * Cb, a);\n" "}\n" "void main()\n" "{\n" " vec2 correctedUv = vec2(vTexCoord.x, 1.0 - vTexCoord.y);\n" " ivec2 decodedSize = ivec2(max(uDecodedVideoResolution, vec2(1.0, 1.0)));\n" " ivec2 outputCoord = clamp(ivec2(correctedUv * vec2(decodedSize)), ivec2(0, 0), decodedSize - ivec2(1, 1));\n" " ivec2 packedSize = ivec2(max(uPackedVideoResolution, vec2(1.0, 1.0)));\n" " ivec2 packedCoord = ivec2(clamp(outputCoord.x / 2, 0, packedSize.x - 1), clamp(outputCoord.y, 0, packedSize.y - 1));\n" " vec4 macroPixel = texelFetch(uPackedVideoInput, packedCoord, 0);\n" " float ySample = (outputCoord.x & 1) != 0 ? macroPixel.a : macroPixel.g;\n" " fragColor = rec709YCbCr2rgba(ySample, macroPixel.b, macroPixel.r, 1.0);\n" "}\n"; void CopyErrorMessage(const std::string& message, int errorMessageSize, char* errorMessage) { if (!errorMessage || errorMessageSize <= 0) return; strncpy_s(errorMessage, errorMessageSize, message.c_str(), _TRUNCATE); } std::string NormalizeModeToken(const std::string& value) { std::string normalized; for (unsigned char ch : value) { if (std::isalnum(ch)) normalized.push_back(static_cast(std::tolower(ch))); } return normalized; } bool ResolveConfiguredDisplayMode(const std::string& videoFormat, const std::string& frameRate, BMDDisplayMode& displayMode, std::string& displayModeName) { const std::string formatToken = NormalizeModeToken(videoFormat); const std::string frameToken = NormalizeModeToken(frameRate); const std::string combinedToken = formatToken + frameToken; struct ModeOption { const char* token; BMDDisplayMode mode; const char* displayName; }; static const ModeOption options[] = { { "720p50", bmdModeHD720p50, "720p50" }, { "hd720p50", bmdModeHD720p50, "720p50" }, { "720p5994", bmdModeHD720p5994, "720p59.94" }, { "hd720p5994", bmdModeHD720p5994, "720p59.94" }, { "720p60", bmdModeHD720p60, "720p60" }, { "hd720p60", bmdModeHD720p60, "720p60" }, { "1080i50", bmdModeHD1080i50, "1080i50" }, { "hd1080i50", bmdModeHD1080i50, "1080i50" }, { "1080i5994", bmdModeHD1080i5994, "1080i59.94" }, { "hd1080i5994", bmdModeHD1080i5994, "1080i59.94" }, { "1080i60", bmdModeHD1080i6000, "1080i60" }, { "hd1080i60", bmdModeHD1080i6000, "1080i60" }, { "1080p2398", bmdModeHD1080p2398, "1080p23.98" }, { "hd1080p2398", bmdModeHD1080p2398, "1080p23.98" }, { "1080p24", bmdModeHD1080p24, "1080p24" }, { "hd1080p24", bmdModeHD1080p24, "1080p24" }, { "1080p25", bmdModeHD1080p25, "1080p25" }, { "hd1080p25", bmdModeHD1080p25, "1080p25" }, { "1080p2997", bmdModeHD1080p2997, "1080p29.97" }, { "hd1080p2997", bmdModeHD1080p2997, "1080p29.97" }, { "1080p30", bmdModeHD1080p30, "1080p30" }, { "hd1080p30", bmdModeHD1080p30, "1080p30" }, { "1080p50", bmdModeHD1080p50, "1080p50" }, { "hd1080p50", bmdModeHD1080p50, "1080p50" }, { "1080p5994", bmdModeHD1080p5994, "1080p59.94" }, { "hd1080p5994", bmdModeHD1080p5994, "1080p59.94" }, { "1080p60", bmdModeHD1080p6000, "1080p60" }, { "hd1080p60", bmdModeHD1080p6000, "1080p60" }, { "2160p2398", bmdMode4K2160p2398, "2160p23.98" }, { "4k2160p2398", bmdMode4K2160p2398, "2160p23.98" }, { "2160p24", bmdMode4K2160p24, "2160p24" }, { "4k2160p24", bmdMode4K2160p24, "2160p24" }, { "2160p25", bmdMode4K2160p25, "2160p25" }, { "4k2160p25", bmdMode4K2160p25, "2160p25" }, { "2160p2997", bmdMode4K2160p2997, "2160p29.97" }, { "4k2160p2997", bmdMode4K2160p2997, "2160p29.97" }, { "2160p30", bmdMode4K2160p30, "2160p30" }, { "4k2160p30", bmdMode4K2160p30, "2160p30" }, { "2160p50", bmdMode4K2160p50, "2160p50" }, { "4k2160p50", bmdMode4K2160p50, "2160p50" }, { "2160p5994", bmdMode4K2160p5994, "2160p59.94" }, { "4k2160p5994", bmdMode4K2160p5994, "2160p59.94" }, { "2160p60", bmdMode4K2160p60, "2160p60" }, { "4k2160p60", bmdMode4K2160p60, "2160p60" } }; for (const ModeOption& option : options) { if (combinedToken == option.token || (frameToken.empty() && formatToken == option.token)) { displayMode = option.mode; displayModeName = option.displayName; return true; } } return false; } class ScopedGlShader { public: explicit ScopedGlShader(GLuint shader = 0) : mShader(shader) {} ~ScopedGlShader() { reset(); } ScopedGlShader(const ScopedGlShader&) = delete; ScopedGlShader& operator=(const ScopedGlShader&) = delete; GLuint get() const { return mShader; } GLuint release() { GLuint shader = mShader; mShader = 0; return shader; } void reset(GLuint shader = 0) { if (mShader != 0) glDeleteShader(mShader); mShader = shader; } private: GLuint mShader; }; class ScopedGlProgram { public: explicit ScopedGlProgram(GLuint program = 0) : mProgram(program) {} ~ScopedGlProgram() { reset(); } ScopedGlProgram(const ScopedGlProgram&) = delete; ScopedGlProgram& operator=(const ScopedGlProgram&) = delete; GLuint get() const { return mProgram; } GLuint release() { GLuint program = mProgram; mProgram = 0; return program; } void reset(GLuint program = 0) { if (mProgram != 0) glDeleteProgram(mProgram); mProgram = program; } private: GLuint mProgram; }; std::size_t AlignStd140(std::size_t offset, std::size_t alignment) { const std::size_t mask = alignment - 1; return (offset + mask) & ~mask; } template void AppendStd140Value(std::vector& buffer, std::size_t alignment, const TValue& value) { const std::size_t offset = AlignStd140(buffer.size(), alignment); if (buffer.size() < offset + sizeof(TValue)) buffer.resize(offset + sizeof(TValue), 0); std::memcpy(buffer.data() + offset, &value, sizeof(TValue)); } void AppendStd140Float(std::vector& buffer, float value) { AppendStd140Value(buffer, 4, value); } void AppendStd140Int(std::vector& buffer, int value) { AppendStd140Value(buffer, 4, value); } void AppendStd140Vec2(std::vector& buffer, float x, float y) { const std::size_t offset = AlignStd140(buffer.size(), 8); if (buffer.size() < offset + sizeof(float) * 2) buffer.resize(offset + sizeof(float) * 2, 0); float values[2] = { x, y }; std::memcpy(buffer.data() + offset, values, sizeof(values)); } void AppendStd140Vec4(std::vector& buffer, float x, float y, float z, float w) { const std::size_t offset = AlignStd140(buffer.size(), 16); if (buffer.size() < offset + sizeof(float) * 4) buffer.resize(offset + sizeof(float) * 4, 0); float values[4] = { x, y, z, w }; std::memcpy(buffer.data() + offset, values, sizeof(values)); } } OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) : hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC), mCaptureDelegate(NULL), mPlayoutDelegate(NULL), mDLInput(NULL), mDLOutput(NULL), mDLKeyer(NULL), mPlayoutAllocator(NULL), mFrameWidth(0), mFrameHeight(0), mDisplayModeName("1080p59.94"), mHasNoInputSource(true), mDeckLinkSupportsInternalKeying(false), mDeckLinkSupportsExternalKeying(false), mDeckLinkKeyerInterfaceAvailable(false), mDeckLinkExternalKeyingActive(false), mFastTransferExtensionAvailable(false), mCaptureTexture(0), mDecodedTexture(0), mLayerTempTexture(0), mFBOTexture(0), mUnpinnedTextureBuffer(0), mDecodeFrameBuf(0), mLayerTempFrameBuf(0), mIdFrameBuf(0), mIdColorBuf(0), mIdDepthBuf(0), mFullscreenVAO(0), mGlobalParamsUBO(0), mDecodeProgram(0), mDecodeVertexShader(0), mDecodeFragmentShader(0), mGlobalParamsUBOSize(0), mViewWidth(0), mViewHeight(0), mTemporalHistoryNeedsReset(true) { InitializeCriticalSection(&pMutex); mRuntimeHost = std::make_unique(); mControlServer = std::make_unique(); mOscServer = std::make_unique(); } OpenGLComposite::~OpenGLComposite() { // Cleanup for Capture if (mDLInput != NULL) { mDLInput->SetCallback(NULL); mDLInput->Release(); mDLInput = NULL; } if (mCaptureDelegate != NULL) { mCaptureDelegate->Release(); mCaptureDelegate = NULL; } // Cleanup for Playout while (!mDLOutputVideoFrameQueue.empty()) { IDeckLinkMutableVideoFrame* frameToRelease = mDLOutputVideoFrameQueue.front(); if (frameToRelease != NULL) { frameToRelease->Release(); frameToRelease = NULL; } mDLOutputVideoFrameQueue.pop_front(); } if (mDLOutput != NULL) { if (mDLKeyer != NULL) { mDLKeyer->Disable(); mDLKeyer->Release(); mDLKeyer = NULL; } mDLOutput->SetScheduledFrameCompletionCallback(NULL); mDLOutput->Release(); mDLOutput = NULL; } if (mPlayoutDelegate != NULL) { mPlayoutDelegate->Release(); mPlayoutDelegate = NULL; } if (mPlayoutAllocator != NULL) { mPlayoutAllocator->Release(); mPlayoutAllocator = NULL; } if (mFullscreenVAO != 0) glDeleteVertexArrays(1, &mFullscreenVAO); if (mGlobalParamsUBO != 0) glDeleteBuffers(1, &mGlobalParamsUBO); if (mDecodeFrameBuf != 0) glDeleteFramebuffers(1, &mDecodeFrameBuf); if (mLayerTempFrameBuf != 0) glDeleteFramebuffers(1, &mLayerTempFrameBuf); if (mIdFrameBuf != 0) glDeleteFramebuffers(1, &mIdFrameBuf); if (mIdColorBuf != 0) glDeleteRenderbuffers(1, &mIdColorBuf); if (mIdDepthBuf != 0) glDeleteRenderbuffers(1, &mIdDepthBuf); if (mCaptureTexture != 0) glDeleteTextures(1, &mCaptureTexture); if (mDecodedTexture != 0) glDeleteTextures(1, &mDecodedTexture); if (mLayerTempTexture != 0) glDeleteTextures(1, &mLayerTempTexture); if (mFBOTexture != 0) glDeleteTextures(1, &mFBOTexture); if (mUnpinnedTextureBuffer != 0) glDeleteBuffers(1, &mUnpinnedTextureBuffer); destroyTemporalHistoryResources(); destroyLayerPrograms(); destroyDecodeShaderProgram(); if (mOscServer) mOscServer->Stop(); if (mControlServer) mControlServer->Stop(); DeleteCriticalSection(&pMutex); } bool OpenGLComposite::InitDeckLink() { bool bSuccess = false; IDeckLinkIterator* pDLIterator = NULL; IDeckLink* pDL = NULL; IDeckLinkProfileAttributes* deckLinkAttributes = NULL; IDeckLinkDisplayModeIterator* pDLDisplayModeIterator = NULL; IDeckLinkDisplayMode* pDLDisplayMode = NULL; BMDDisplayMode displayMode = bmdModeHD1080p5994; // mode to use for capture and playout std::string displayModeName = "1080p59.94"; int outputFrameRowBytes; HRESULT result; if (mRuntimeHost && mRuntimeHost->GetRepoRoot().empty()) { std::string runtimeError; if (!mRuntimeHost->Initialize(runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK); return false; } } if (mRuntimeHost) { if (!ResolveConfiguredDisplayMode(mRuntimeHost->GetVideoFormat(), mRuntimeHost->GetFrameRate(), displayMode, displayModeName)) { const std::string error = "Unsupported DeckLink video format/frameRate in config/runtime-host.json: " + mRuntimeHost->GetVideoFormat() + " / " + mRuntimeHost->GetFrameRate(); MessageBoxA(NULL, error.c_str(), "DeckLink mode configuration error", MB_OK); return false; } } mDisplayModeName = displayModeName; result = CoCreateInstance(CLSID_CDeckLinkIterator, NULL, CLSCTX_ALL, IID_IDeckLinkIterator, (void**)&pDLIterator); if (FAILED(result)) { MessageBox(NULL, _T("Please install the Blackmagic DeckLink drivers to use the features of this application."), _T("This application requires the DeckLink drivers installed."), MB_OK); return false; } while (pDLIterator->Next(&pDL) == S_OK) { int64_t duplexMode; bool supportsInternalKeying = false; bool supportsExternalKeying = false; std::string modelName; if (result = pDL->QueryInterface(IID_IDeckLinkProfileAttributes, (void**)&deckLinkAttributes) != S_OK) { printf("Could not obtain the IDeckLinkProfileAttributes interface - result %08x\n", result); pDL->Release(); pDL = NULL; continue; } result = deckLinkAttributes->GetInt(BMDDeckLinkDuplex, &duplexMode); BOOL attributeFlag = FALSE; if (deckLinkAttributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &attributeFlag) == S_OK) supportsInternalKeying = (attributeFlag != FALSE); attributeFlag = FALSE; if (deckLinkAttributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &attributeFlag) == S_OK) supportsExternalKeying = (attributeFlag != FALSE); BSTR modelNameBstr = NULL; if (deckLinkAttributes->GetString(BMDDeckLinkModelName, &modelNameBstr) == S_OK && modelNameBstr != NULL) { const int requiredBytes = WideCharToMultiByte(CP_UTF8, 0, modelNameBstr, -1, NULL, 0, NULL, NULL); if (requiredBytes > 1) { std::vector utf8Name(static_cast(requiredBytes), '\0'); if (WideCharToMultiByte(CP_UTF8, 0, modelNameBstr, -1, utf8Name.data(), requiredBytes, NULL, NULL) > 0) modelName.assign(utf8Name.data()); } SysFreeString(modelNameBstr); } deckLinkAttributes->Release(); deckLinkAttributes = NULL; if (result != S_OK || duplexMode == bmdDuplexInactive) { pDL->Release(); pDL = NULL; continue; } // Use a full duplex device as capture and playback, or half-duplex device // as capture or playback. bool inputUsed = false; if (!mDLInput && pDL->QueryInterface(IID_IDeckLinkInput, (void**)&mDLInput) == S_OK) inputUsed = true; if (!mDLOutput && (!inputUsed || (duplexMode == bmdDuplexFull))) { if (pDL->QueryInterface(IID_IDeckLinkOutput, (void**)&mDLOutput) != S_OK) mDLOutput = NULL; else { mDeckLinkOutputModelName = modelName; mDeckLinkSupportsInternalKeying = supportsInternalKeying; mDeckLinkSupportsExternalKeying = supportsExternalKeying; } } pDL->Release(); pDL = NULL; if (mDLOutput && mDLInput) break; } if (! mDLOutput || ! mDLInput) { MessageBox(NULL, _T("Expected both Input and Output DeckLink devices"), _T("This application requires two DeckLink devices."), MB_OK); goto error; } if (mDLOutput->GetDisplayModeIterator(&pDLDisplayModeIterator) != S_OK) { MessageBox(NULL, _T("Cannot get Display Mode Iterator."), _T("DeckLink error."), MB_OK); goto error; } while (pDLDisplayModeIterator->Next(&pDLDisplayMode) == S_OK) { if (pDLDisplayMode->GetDisplayMode() == displayMode) break; pDLDisplayMode->Release(); pDLDisplayMode = NULL; } pDLDisplayModeIterator->Release(); pDLDisplayModeIterator = NULL; if (pDLDisplayMode == NULL) { const std::string error = "Cannot get specified BMDDisplayMode for configured mode: " + displayModeName; MessageBoxA(NULL, error.c_str(), "DeckLink error.", MB_OK); goto error; } mFrameWidth = pDLDisplayMode->GetWidth(); mFrameHeight = pDLDisplayMode->GetHeight(); if (! CheckOpenGLExtensions()) goto error; if (! InitOpenGLState()) goto error; if (mRuntimeHost) { mDeckLinkStatusMessage = mDeckLinkOutputModelName.empty() ? "DeckLink output device selected." : ("Selected output device: " + mDeckLinkOutputModelName); mRuntimeHost->SetDeckLinkOutputStatus( mDeckLinkOutputModelName, mDeckLinkSupportsInternalKeying, mDeckLinkSupportsExternalKeying, mDeckLinkKeyerInterfaceAvailable, mRuntimeHost->ExternalKeyingEnabled(), mDeckLinkExternalKeyingActive, mDeckLinkStatusMessage); } pDLDisplayMode->GetFrameRate(&mFrameDuration, &mFrameTimescale); // Resize window to match video frame, but scale large formats down by half for viewing if (mFrameWidth < 1920) resizeWindow(mFrameWidth, mFrameHeight); else resizeWindow(mFrameWidth / 2, mFrameHeight / 2); if (mFastTransferExtensionAvailable) { // Initialize fast video frame transfers if (! VideoFrameTransfer::initialize(mFrameWidth, mFrameHeight, mCaptureTexture, mFBOTexture)) { MessageBox(NULL, _T("Cannot initialize video transfers."), _T("VideoFrameTransfer error."), MB_OK); goto error; } } { // Use custom allocators so we pin only once then recycle them CComPtr captureAllocator(new (std::nothrow) InputAllocatorPool(hGLDC, hGLRC)); if (mDLInput->EnableVideoInputWithAllocatorProvider(displayMode, bmdFormat8BitYUV, bmdVideoInputFlagDefault, captureAllocator) != S_OK) goto error; } mCaptureDelegate = new CaptureDelegate(this); if (mDLInput->SetCallback(mCaptureDelegate) != S_OK) goto error; if (mDLOutput->RowBytesForPixelFormat(bmdFormat8BitBGRA, mFrameWidth, &outputFrameRowBytes) != S_OK) goto error; // Use a custom allocator so we pin only once then recycle them mPlayoutAllocator = new PinnedMemoryAllocator(hGLDC, hGLRC, VideoFrameTransfer::GPUtoCPU, 1, outputFrameRowBytes * mFrameHeight); if (mDLOutput->EnableVideoOutput(displayMode, bmdVideoOutputFlagDefault) != S_OK) goto error; if (mDLOutput->QueryInterface(IID_IDeckLinkKeyer, (void**)&mDLKeyer) == S_OK && mDLKeyer != NULL) mDeckLinkKeyerInterfaceAvailable = true; if (mRuntimeHost && mRuntimeHost->ExternalKeyingEnabled()) { if (!mDeckLinkSupportsExternalKeying) { mDeckLinkStatusMessage = "External keying was requested, but the selected DeckLink output does not report external keying support."; } else if (!mDeckLinkKeyerInterfaceAvailable) { mDeckLinkStatusMessage = "External keying was requested, but the selected DeckLink output does not expose the IDeckLinkKeyer interface."; } else if (mDLKeyer->Enable(TRUE) != S_OK || mDLKeyer->SetLevel(255) != S_OK) { mDeckLinkStatusMessage = "External keying was requested, but enabling the DeckLink keyer failed."; } else { mDeckLinkExternalKeyingActive = true; mDeckLinkStatusMessage = "External keying is active on the selected DeckLink output."; } } else if (mDeckLinkSupportsExternalKeying) { mDeckLinkStatusMessage = "Selected DeckLink output supports external keying. Set enableExternalKeying to true in runtime-host.json to request it."; } if (mRuntimeHost) { mRuntimeHost->SetDeckLinkOutputStatus( mDeckLinkOutputModelName, mDeckLinkSupportsInternalKeying, mDeckLinkSupportsExternalKeying, mDeckLinkKeyerInterfaceAvailable, mRuntimeHost->ExternalKeyingEnabled(), mDeckLinkExternalKeyingActive, mDeckLinkStatusMessage); } // Create a queue of 10 IDeckLinkMutableVideoFrame objects to use for scheduling output video frames. // The ScheduledFrameCompleted() callback will immediately schedule a new frame using the next video frame from this queue. for (int i = 0; i < 10; i++) { // The frame read back from the GPU frame buffer and used for the playout video frame is in BGRA format. // The BGRA frame will be converted on playout to YCbCr either in hardware on most DeckLink cards or in software // within the DeckLink API for DeckLink devices without this hardware conversion. // If you want RGB 4:4:4 format to be played out "over the wire" in SDI, turn on the "Use 4:4:4 SDI" in the control // panel or turn on the bmdDeckLinkConfig444SDIVideoOutput flag using the IDeckLinkConfiguration interface. IDeckLinkMutableVideoFrame* outputFrame; IDeckLinkVideoBuffer* outputFrameBuffer = NULL; if (mPlayoutAllocator->AllocateVideoBuffer(&outputFrameBuffer) != S_OK) goto error; if (mDLOutput->CreateVideoFrameWithBuffer(mFrameWidth, mFrameHeight, outputFrameRowBytes, bmdFormat8BitBGRA, bmdFrameFlagFlipVertical, outputFrameBuffer, &outputFrame) != S_OK) goto error; mDLOutputVideoFrameQueue.push_back(outputFrame); } mPlayoutDelegate = new PlayoutDelegate(this); if (mPlayoutDelegate == NULL) goto error; if (mDLOutput->SetScheduledFrameCompletionCallback(mPlayoutDelegate) != S_OK) goto error; bSuccess = true; error: if (!bSuccess) { if (mDLKeyer != NULL) { mDLKeyer->Disable(); mDLKeyer->Release(); mDLKeyer = NULL; mDeckLinkExternalKeyingActive = false; } if (mDLInput != NULL) { mDLInput->Release(); mDLInput = NULL; } if (mDLOutput != NULL) { mDLOutput->Release(); mDLOutput = NULL; } } if (pDL != NULL) { pDL->Release(); pDL = NULL; } if (pDLDisplayMode != NULL) { pDLDisplayMode->Release(); pDLDisplayMode = NULL; } if (pDLIterator != NULL) { pDLIterator->Release(); pDLIterator = NULL; } return bSuccess; } void OpenGLComposite::paintGL() { if (!TryEnterCriticalSection(&pMutex)) { ValidateRect(hGLWnd, NULL); return; } // The DeckLink API provides IDeckLinkGLScreenPreviewHelper as a convenient way to view the playout video frames // in a window. However, it performs a copy from host memory to the GPU which is wasteful in this case since // we already have the rendered frame to be played out sitting in the GPU in the mIdFrameBuf frame buffer. // Copy the off-screen frame buffer to the on-screen frame buffer while preserving the // incoming video aspect ratio. Any extra window area is cleared to black. int destWidth = mViewWidth; int destHeight = mViewHeight; int destX = 0; int destY = 0; if (mFrameWidth > 0 && mFrameHeight > 0 && mViewWidth > 0 && mViewHeight > 0) { const double frameAspect = static_cast(mFrameWidth) / static_cast(mFrameHeight); const double viewAspect = static_cast(mViewWidth) / static_cast(mViewHeight); if (viewAspect > frameAspect) { destHeight = mViewHeight; destWidth = static_cast(destHeight * frameAspect + 0.5); destX = (mViewWidth - destWidth) / 2; } else { destWidth = mViewWidth; destHeight = static_cast(destWidth / frameAspect + 0.5); destY = (mViewHeight - destHeight) / 2; } } glBindFramebuffer(GL_READ_FRAMEBUFFER, mIdFrameBuf); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0); glViewport(0, 0, mViewWidth, mViewHeight); glClearColor(0.0f, 0.0f, 0.0f, 1.0f); glClear(GL_COLOR_BUFFER_BIT); glBlitFramebuffer(0, 0, mFrameWidth, mFrameHeight, destX, destY, destX + destWidth, destY + destHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); SwapBuffers(hGLDC); ValidateRect(hGLWnd, NULL); LeaveCriticalSection(&pMutex); } void OpenGLComposite::resizeGL(WORD width, WORD height) { // We don't set the project or model matrices here since the window data is copied directly from // an off-screen FBO in paintGL(). Just save the width and height for use in paintGL(). mViewWidth = width; mViewHeight = height; } void OpenGLComposite::resizeWindow(int width, int height) { RECT r; if (GetWindowRect(hGLWnd, &r)) { SetWindowPos(hGLWnd, HWND_TOP, r.left, r.top, r.left + width, r.top + height, 0); } } bool OpenGLComposite::InitOpenGLState() { if (! ResolveGLExtensions()) return false; std::string runtimeError; if (mRuntimeHost->GetRepoRoot().empty() && !mRuntimeHost->Initialize(runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK); return false; } ControlServer::Callbacks callbacks; callbacks.getStateJson = [this]() { return GetRuntimeStateJson(); }; callbacks.addLayer = [this](const std::string& shaderId, std::string& error) { return AddLayer(shaderId, error); }; callbacks.removeLayer = [this](const std::string& layerId, std::string& error) { return RemoveLayer(layerId, error); }; callbacks.moveLayer = [this](const std::string& layerId, int direction, std::string& error) { return MoveLayer(layerId, direction, error); }; callbacks.moveLayerToIndex = [this](const std::string& layerId, std::size_t targetIndex, std::string& error) { return MoveLayerToIndex(layerId, targetIndex, error); }; callbacks.setLayerBypass = [this](const std::string& layerId, bool bypassed, std::string& error) { return SetLayerBypass(layerId, bypassed, error); }; callbacks.setLayerShader = [this](const std::string& layerId, const std::string& shaderId, std::string& error) { return SetLayerShader(layerId, shaderId, error); }; callbacks.updateLayerParameter = [this](const std::string& layerId, const std::string& parameterId, const std::string& valueJson, std::string& error) { return UpdateLayerParameterJson(layerId, parameterId, valueJson, error); }; callbacks.resetLayerParameters = [this](const std::string& layerId, std::string& error) { return ResetLayerParameters(layerId, error); }; callbacks.saveStackPreset = [this](const std::string& presetName, std::string& error) { return SaveStackPreset(presetName, error); }; callbacks.loadStackPreset = [this](const std::string& presetName, std::string& error) { return LoadStackPreset(presetName, error); }; callbacks.reloadShader = [this](std::string& error) { if (!ReloadShader()) { error = "Shader reload failed. See native app status for details."; return false; } return true; }; if (!mControlServer->Start(mRuntimeHost->GetUiRoot(), mRuntimeHost->GetDocsRoot(), mRuntimeHost->GetServerPort(), callbacks, runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "Local control server failed to start", MB_OK); return false; } mRuntimeHost->SetServerPort(mControlServer->GetPort()); OscServer::Callbacks oscCallbacks; oscCallbacks.updateParameter = [this](const std::string& layerKey, const std::string& parameterKey, const std::string& valueJson, std::string& error) { return UpdateLayerParameterByControlKeyJson(layerKey, parameterKey, valueJson, error); }; if (mRuntimeHost->GetOscPort() > 0 && !mOscServer->Start(mRuntimeHost->GetOscPort(), oscCallbacks, runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "OSC control server failed to start", MB_OK); return false; } // Prepare the runtime shader program generated from the active shader package. char compilerErrorMessage[1024]; if (! compileDecodeShader(sizeof(compilerErrorMessage), compilerErrorMessage)) { MessageBoxA(NULL, compilerErrorMessage, "OpenGL decode shader failed to load or compile", MB_OK); return false; } if (! compileLayerPrograms(sizeof(compilerErrorMessage), compilerErrorMessage)) { MessageBoxA(NULL, compilerErrorMessage, "OpenGL shader failed to load or compile", MB_OK); return false; } resetTemporalHistoryState(); glClearColor( 0.0f, 0.0f, 0.0f, 0.5f ); // Black background glDisable(GL_DEPTH_TEST); if (! mFastTransferExtensionAvailable) { glGenBuffers(1, &mUnpinnedTextureBuffer); } // Setup the texture which will hold the captured video frame pixels glGenTextures(1, &mCaptureTexture); glBindTexture(GL_TEXTURE_2D, mCaptureTexture); // Parameters to control how texels are sampled from the texture glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); // Create texture with empty data, we will update it using glTexSubImage2D each frame. // The captured video is YCbCr 4:2:2 packed into a UYVY macropixel. OpenGL has no YCbCr format // so treat it as RGBA 4:4:4:4 by halving the width and using GL_RGBA internal format. glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mFrameWidth/2, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glBindTexture(GL_TEXTURE_2D, 0); glGenTextures(1, &mDecodedTexture); glBindTexture(GL_TEXTURE_2D, mDecodedTexture); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mFrameWidth, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glBindTexture(GL_TEXTURE_2D, 0); glGenTextures(1, &mLayerTempTexture); glBindTexture(GL_TEXTURE_2D, mLayerTempTexture); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mFrameWidth, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glBindTexture(GL_TEXTURE_2D, 0); // Create Frame Buffer Object (FBO) to perform off-screen rendering of scene. // This allows the render to be done on a framebuffer with width and height exactly matching the video format. glGenFramebuffers(1, &mDecodeFrameBuf); glGenFramebuffers(1, &mLayerTempFrameBuf); glGenFramebuffers(1, &mIdFrameBuf); glGenRenderbuffers(1, &mIdColorBuf); glGenRenderbuffers(1, &mIdDepthBuf); glGenVertexArrays(1, &mFullscreenVAO); glGenBuffers(1, &mGlobalParamsUBO); glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf); glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mDecodedTexture, 0); GLenum glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER); if (glStatus != GL_FRAMEBUFFER_COMPLETE) { MessageBox(NULL, _T("Cannot initialize decode framebuffer."), _T("OpenGL initialization error."), MB_OK); return false; } glBindFramebuffer(GL_FRAMEBUFFER, mLayerTempFrameBuf); glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mLayerTempTexture, 0); glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER); if (glStatus != GL_FRAMEBUFFER_COMPLETE) { MessageBox(NULL, _T("Cannot initialize layer framebuffer."), _T("OpenGL initialization error."), MB_OK); return false; } glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf); // Texture for FBO glGenTextures(1, &mFBOTexture); glBindTexture(GL_TEXTURE_2D, mFBOTexture); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mFrameWidth, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); // Attach a depth buffer glBindRenderbuffer(GL_RENDERBUFFER, mIdDepthBuf); glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT24, mFrameWidth, mFrameHeight); glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT_EXT, GL_RENDERBUFFER, mIdDepthBuf); // Attach the texture which stores the playback image glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mFBOTexture, 0); glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER); if (glStatus != GL_FRAMEBUFFER_COMPLETE) { MessageBox(NULL, _T("Cannot initialize framebuffer."), _T("OpenGL initialization error."), MB_OK); return false; } glBindTexture(GL_TEXTURE_2D, 0); glBindRenderbuffer(GL_RENDERBUFFER, 0); glBindFramebuffer(GL_FRAMEBUFFER, 0); glBindVertexArray(mFullscreenVAO); glBindVertexArray(0); glBindBuffer(GL_UNIFORM_BUFFER, mGlobalParamsUBO); glBufferData(GL_UNIFORM_BUFFER, 1024, NULL, GL_DYNAMIC_DRAW); glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mGlobalParamsUBO); glBindBuffer(GL_UNIFORM_BUFFER, 0); broadcastRuntimeState(); return true; } // // Update the captured video frame texture // void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource) { mHasNoInputSource = hasNoInputSource; if (mRuntimeHost) mRuntimeHost->SetSignalStatus(!hasNoInputSource, mFrameWidth, mFrameHeight, mDisplayModeName); if (mHasNoInputSource) return; // don't transfer texture when there's no input long textureSize = inputFrame->GetRowBytes() * inputFrame->GetHeight(); IDeckLinkVideoBuffer* inputFrameBuffer = NULL; void* videoPixels; if (inputFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&inputFrameBuffer) != S_OK) return; if (inputFrameBuffer->StartAccess(bmdBufferAccessRead) != S_OK) { inputFrameBuffer->Release(); return; } inputFrameBuffer->GetBytes(&videoPixels); EnterCriticalSection(&pMutex); wglMakeCurrent( hGLDC, hGLRC ); // make OpenGL context current in this thread if (mFastTransferExtensionAvailable) { CComQIPtr allocator(inputFrameBuffer); if (!allocator || !allocator->transferFrame(videoPixels, mCaptureTexture)) OutputDebugStringA("Capture: transferFrame() failed\n"); allocator->waitForTransferComplete(videoPixels); } else { // Use a straightforward texture buffer glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mUnpinnedTextureBuffer); glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, videoPixels, GL_DYNAMIC_DRAW); glBindTexture(GL_TEXTURE_2D, mCaptureTexture); // NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mFrameWidth/2, mFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glBindTexture(GL_TEXTURE_2D, 0); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); } wglMakeCurrent( NULL, NULL ); LeaveCriticalSection(&pMutex); inputFrameBuffer->EndAccess(bmdBufferAccessRead); inputFrameBuffer->Release(); } // Render the live video texture through the runtime shader into the off-screen framebuffer. // Read the result back from the frame buffer and schedule it for playout. void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult) { EnterCriticalSection(&pMutex); // Get the first frame from the queue IDeckLinkMutableVideoFrame* outputVideoFrame = mDLOutputVideoFrameQueue.front(); mDLOutputVideoFrameQueue.push_back(outputVideoFrame); mDLOutputVideoFrameQueue.pop_front(); // make GL context current in this thread wglMakeCurrent( hGLDC, hGLRC ); // Draw the effect output to the off-screen framebuffer. const auto renderStartTime = std::chrono::steady_clock::now(); if (mFastTransferExtensionAvailable) VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::GPUtoCPU); glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf); renderEffect(); glFlush(); if (mFastTransferExtensionAvailable) VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::GPUtoCPU); const auto renderEndTime = std::chrono::steady_clock::now(); if (mRuntimeHost) { const double frameBudgetMilliseconds = mFrameTimescale != 0 ? (static_cast(mFrameDuration) * 1000.0) / static_cast(mFrameTimescale) : 0.0; const double renderMilliseconds = std::chrono::duration_cast>(renderEndTime - renderStartTime).count(); mRuntimeHost->SetPerformanceStats(frameBudgetMilliseconds, renderMilliseconds); } if (mRuntimeHost) mRuntimeHost->AdvanceFrame(); IDeckLinkVideoBuffer* outputVideoFrameBuffer; if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK) { LeaveCriticalSection(&pMutex); return; } if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK) { outputVideoFrameBuffer->Release(); LeaveCriticalSection(&pMutex); return; } void* pFrame; outputVideoFrameBuffer->GetBytes(&pFrame); if (mFastTransferExtensionAvailable) { // Finished with mCaptureTexture VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); if (! mPlayoutAllocator->transferFrame(pFrame, mFBOTexture)) OutputDebugStringA("Playback: transferFrame() failed\n"); paintGL(); // Wait for transfer to system memory to complete mPlayoutAllocator->waitForTransferComplete(pFrame); } else { glBindFramebuffer(GL_READ_FRAMEBUFFER, mIdFrameBuf); glReadPixels(0, 0, mFrameWidth, mFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame); paintGL(); } outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite); outputVideoFrameBuffer->Release(); // If the last completed frame was late or dropped, bump the scheduled time further into the future if (completionResult == bmdOutputFrameDisplayedLate || completionResult == bmdOutputFrameDropped) mTotalPlayoutFrames += 2; // Schedule the next frame for playout HRESULT hr = mDLOutput->ScheduleVideoFrame(outputVideoFrame, (mTotalPlayoutFrames * mFrameDuration), mFrameDuration, mFrameTimescale); if (SUCCEEDED(hr)) mTotalPlayoutFrames++; wglMakeCurrent( NULL, NULL ); LeaveCriticalSection(&pMutex); } bool OpenGLComposite::Start() { mTotalPlayoutFrames = 0; // Preroll frames for (unsigned i = 0; i < 5; i++) { // Take each video frame from the front of the queue and move it to the back IDeckLinkMutableVideoFrame* outputVideoFrame = mDLOutputVideoFrameQueue.front(); mDLOutputVideoFrameQueue.push_back(outputVideoFrame); mDLOutputVideoFrameQueue.pop_front(); // Start with a black frame for playout IDeckLinkVideoBuffer* outputVideoFrameBuffer; if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK) return false; if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK) { outputVideoFrameBuffer->Release(); return false; } void* pFrame; outputVideoFrameBuffer->GetBytes((void**)&pFrame); memset(pFrame, 0, outputVideoFrame->GetRowBytes() * mFrameHeight); // 0 is black in RGBA format outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite); outputVideoFrameBuffer->Release(); if (mDLOutput->ScheduleVideoFrame(outputVideoFrame, (mTotalPlayoutFrames * mFrameDuration), mFrameDuration, mFrameTimescale) != S_OK) return false; mTotalPlayoutFrames++; } mDLInput->StartStreams(); mDLOutput->StartScheduledPlayback(0, mFrameTimescale, 1.0); return true; } bool OpenGLComposite::Stop() { if (mOscServer) mOscServer->Stop(); if (mControlServer) mControlServer->Stop(); if (mDLKeyer != NULL) { mDLKeyer->Disable(); mDeckLinkExternalKeyingActive = false; if (mRuntimeHost) { mRuntimeHost->SetDeckLinkOutputStatus( mDeckLinkOutputModelName, mDeckLinkSupportsInternalKeying, mDeckLinkSupportsExternalKeying, mDeckLinkKeyerInterfaceAvailable, mRuntimeHost->ExternalKeyingEnabled(), mDeckLinkExternalKeyingActive, "External keying has been disabled."); } } mDLInput->StopStreams(); mDLInput->DisableVideoInput(); mDLOutput->StopScheduledPlayback(0, NULL, 0); mDLOutput->DisableVideoOutput(); return true; } bool OpenGLComposite::ReloadShader() { char compilerErrorMessage[1024]; EnterCriticalSection(&pMutex); wglMakeCurrent(hGLDC, hGLRC); bool success = compileLayerPrograms(sizeof(compilerErrorMessage), compilerErrorMessage); if (mRuntimeHost) mRuntimeHost->ClearReloadRequest(); wglMakeCurrent(NULL, NULL); LeaveCriticalSection(&pMutex); if (!success) { if (mRuntimeHost) mRuntimeHost->SetCompileStatus(false, compilerErrorMessage); MessageBoxA(NULL, compilerErrorMessage, "Slang shader reload failed", MB_OK); } else { if (mRuntimeHost) mRuntimeHost->SetCompileStatus(true, "Shader compiled successfully."); broadcastRuntimeState(); } return success; } bool OpenGLComposite::compileSingleLayerProgram(const RuntimeRenderState& state, LayerProgram& layerProgram, int errorMessageSize, char* errorMessage) { GLsizei errorBufferSize = 0; GLint compileResult = GL_FALSE; GLint linkResult = GL_FALSE; std::string fragmentShaderSource; std::string loadError; std::vector textureBindings; const char* vertexSource = kVertexShaderSource; if (!mRuntimeHost->BuildLayerFragmentShaderSource(state.layerId, fragmentShaderSource, loadError)) { CopyErrorMessage(loadError, errorMessageSize, errorMessage); return false; } const char* fragmentSource = fragmentShaderSource.c_str(); ScopedGlShader newVertexShader(glCreateShader(GL_VERTEX_SHADER)); glShaderSource(newVertexShader.get(), 1, (const GLchar**)&vertexSource, NULL); glCompileShader(newVertexShader.get()); glGetShaderiv(newVertexShader.get(), GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newVertexShader.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } ScopedGlShader newFragmentShader(glCreateShader(GL_FRAGMENT_SHADER)); glShaderSource(newFragmentShader.get(), 1, (const GLchar**)&fragmentSource, NULL); glCompileShader(newFragmentShader.get()); glGetShaderiv(newFragmentShader.get(), GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newFragmentShader.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } ScopedGlProgram newProgram(glCreateProgram()); glAttachShader(newProgram.get(), newVertexShader.get()); glAttachShader(newProgram.get(), newFragmentShader.get()); glLinkProgram(newProgram.get()); glGetProgramiv(newProgram.get(), GL_LINK_STATUS, &linkResult); if (linkResult == GL_FALSE) { glGetProgramInfoLog(newProgram.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } for (const ShaderTextureAsset& textureAsset : state.textureAssets) { LayerProgram::TextureBinding textureBinding; textureBinding.samplerName = textureAsset.id; textureBinding.sourcePath = textureAsset.path; if (!loadTextureAsset(textureAsset, textureBinding.texture, loadError)) { for (LayerProgram::TextureBinding& loadedTexture : textureBindings) { if (loadedTexture.texture != 0) glDeleteTextures(1, &loadedTexture.texture); } CopyErrorMessage(loadError, errorMessageSize, errorMessage); return false; } textureBindings.push_back(textureBinding); } const GLuint globalParamsIndex = glGetUniformBlockIndex(newProgram.get(), "GlobalParams"); if (globalParamsIndex != GL_INVALID_INDEX) glUniformBlockBinding(newProgram.get(), globalParamsIndex, kGlobalParamsBindingPoint); const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; const GLuint shaderTextureBase = kSourceHistoryTextureUnitBase + historyCap + historyCap; glUseProgram(newProgram.get()); const GLint videoInputLocation = glGetUniformLocation(newProgram.get(), "gVideoInput"); if (videoInputLocation >= 0) glUniform1i(videoInputLocation, static_cast(kDecodedVideoTextureUnit)); for (unsigned index = 0; index < historyCap; ++index) { const std::string sourceSamplerName = "gSourceHistory" + std::to_string(index); const GLint sourceSamplerLocation = glGetUniformLocation(newProgram.get(), sourceSamplerName.c_str()); if (sourceSamplerLocation >= 0) glUniform1i(sourceSamplerLocation, static_cast(kSourceHistoryTextureUnitBase + index)); const std::string temporalSamplerName = "gTemporalHistory" + std::to_string(index); const GLint temporalSamplerLocation = glGetUniformLocation(newProgram.get(), temporalSamplerName.c_str()); if (temporalSamplerLocation >= 0) glUniform1i(temporalSamplerLocation, static_cast(kSourceHistoryTextureUnitBase + historyCap + index)); } for (std::size_t index = 0; index < textureBindings.size(); ++index) { const GLint textureSamplerLocation = glGetUniformLocation(newProgram.get(), textureBindings[index].samplerName.c_str()); if (textureSamplerLocation >= 0) glUniform1i(textureSamplerLocation, static_cast(shaderTextureBase + static_cast(index))); } glUseProgram(0); layerProgram.layerId = state.layerId; layerProgram.shaderId = state.shaderId; layerProgram.program = newProgram.release(); layerProgram.vertexShader = newVertexShader.release(); layerProgram.fragmentShader = newFragmentShader.release(); layerProgram.textureBindings.swap(textureBindings); return true; } bool OpenGLComposite::compileLayerPrograms(int errorMessageSize, char* errorMessage) { const std::vector layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mFrameWidth, mFrameHeight) : std::vector(); std::string temporalError; if (!validateTemporalTextureUnitBudget(layerStates, temporalError)) { CopyErrorMessage(temporalError, errorMessageSize, errorMessage); return false; } if (!ensureTemporalHistoryResources(layerStates, temporalError)) { CopyErrorMessage(temporalError, errorMessageSize, errorMessage); return false; } std::vector newPrograms; newPrograms.reserve(layerStates.size()); for (const RuntimeRenderState& state : layerStates) { LayerProgram layerProgram; if (!compileSingleLayerProgram(state, layerProgram, errorMessageSize, errorMessage)) { for (LayerProgram& program : newPrograms) destroySingleLayerProgram(program); return false; } newPrograms.push_back(layerProgram); } destroyLayerPrograms(); mLayerPrograms.swap(newPrograms); if (mRuntimeHost) { mRuntimeHost->SetCompileStatus(true, "Shader layers compiled successfully."); mRuntimeHost->ClearReloadRequest(); } return true; } bool OpenGLComposite::compileDecodeShader(int errorMessageSize, char* errorMessage) { GLsizei errorBufferSize = 0; GLint compileResult = GL_FALSE; GLint linkResult = GL_FALSE; const char* vertexSource = kVertexShaderSource; const char* fragmentSource = kDecodeFragmentShaderSource; ScopedGlShader newVertexShader(glCreateShader(GL_VERTEX_SHADER)); glShaderSource(newVertexShader.get(), 1, (const GLchar**)&vertexSource, NULL); glCompileShader(newVertexShader.get()); glGetShaderiv(newVertexShader.get(), GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newVertexShader.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } ScopedGlShader newFragmentShader(glCreateShader(GL_FRAGMENT_SHADER)); glShaderSource(newFragmentShader.get(), 1, (const GLchar**)&fragmentSource, NULL); glCompileShader(newFragmentShader.get()); glGetShaderiv(newFragmentShader.get(), GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newFragmentShader.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } ScopedGlProgram newProgram(glCreateProgram()); glAttachShader(newProgram.get(), newVertexShader.get()); glAttachShader(newProgram.get(), newFragmentShader.get()); glLinkProgram(newProgram.get()); glGetProgramiv(newProgram.get(), GL_LINK_STATUS, &linkResult); if (linkResult == GL_FALSE) { glGetProgramInfoLog(newProgram.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } destroyDecodeShaderProgram(); mDecodeProgram = newProgram.release(); mDecodeVertexShader = newVertexShader.release(); mDecodeFragmentShader = newFragmentShader.release(); return true; } void OpenGLComposite::destroySingleLayerProgram(LayerProgram& layerProgram) { for (LayerProgram::TextureBinding& textureBinding : layerProgram.textureBindings) { if (textureBinding.texture != 0) { glDeleteTextures(1, &textureBinding.texture); textureBinding.texture = 0; } } layerProgram.textureBindings.clear(); if (layerProgram.program != 0) { glDeleteProgram(layerProgram.program); layerProgram.program = 0; } if (layerProgram.fragmentShader != 0) { glDeleteShader(layerProgram.fragmentShader); layerProgram.fragmentShader = 0; } if (layerProgram.vertexShader != 0) { glDeleteShader(layerProgram.vertexShader); layerProgram.vertexShader = 0; } } void OpenGLComposite::destroyLayerPrograms() { for (LayerProgram& layerProgram : mLayerPrograms) destroySingleLayerProgram(layerProgram); mLayerPrograms.clear(); } bool OpenGLComposite::loadTextureAsset(const ShaderTextureAsset& textureAsset, GLuint& textureId, std::string& error) { textureId = 0; HRESULT comInitResult = CoInitializeEx(NULL, COINIT_MULTITHREADED); const bool shouldUninitializeCom = (comInitResult == S_OK || comInitResult == S_FALSE); if (FAILED(comInitResult) && comInitResult != RPC_E_CHANGED_MODE) { error = "Could not initialize COM to load shader texture assets."; return false; } CComPtr imagingFactory; HRESULT result = CoCreateInstance(CLSID_WICImagingFactory, NULL, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&imagingFactory)); if (FAILED(result) || !imagingFactory) { if (shouldUninitializeCom) CoUninitialize(); error = "Could not create a WIC imaging factory to load shader texture assets."; return false; } CComPtr bitmapDecoder; result = imagingFactory->CreateDecoderFromFilename(textureAsset.path.wstring().c_str(), NULL, GENERIC_READ, WICDecodeMetadataCacheOnLoad, &bitmapDecoder); if (FAILED(result) || !bitmapDecoder) { if (shouldUninitializeCom) CoUninitialize(); error = "Could not open shader texture asset: " + textureAsset.path.string(); return false; } CComPtr bitmapFrame; result = bitmapDecoder->GetFrame(0, &bitmapFrame); if (FAILED(result) || !bitmapFrame) { if (shouldUninitializeCom) CoUninitialize(); error = "Could not decode the first frame of shader texture asset: " + textureAsset.path.string(); return false; } CComPtr formatConverter; result = imagingFactory->CreateFormatConverter(&formatConverter); if (FAILED(result) || !formatConverter) { if (shouldUninitializeCom) CoUninitialize(); error = "Could not create a WIC format converter for shader texture asset: " + textureAsset.path.string(); return false; } result = formatConverter->Initialize(bitmapFrame, GUID_WICPixelFormat32bppBGRA, WICBitmapDitherTypeNone, NULL, 0.0, WICBitmapPaletteTypeCustom); if (FAILED(result)) { if (shouldUninitializeCom) CoUninitialize(); error = "Could not convert shader texture asset to BGRA: " + textureAsset.path.string(); return false; } UINT width = 0; UINT height = 0; result = formatConverter->GetSize(&width, &height); if (FAILED(result) || width == 0 || height == 0) { if (shouldUninitializeCom) CoUninitialize(); error = "Shader texture asset has an invalid size: " + textureAsset.path.string(); return false; } const UINT stride = width * 4; std::vector pixels(static_cast(stride) * static_cast(height)); result = formatConverter->CopyPixels(NULL, stride, static_cast(pixels.size()), pixels.data()); if (FAILED(result)) { if (shouldUninitializeCom) CoUninitialize(); error = "Could not read shader texture pixels: " + textureAsset.path.string(); return false; } std::vector flippedPixels(pixels.size()); for (UINT row = 0; row < height; ++row) { const std::size_t srcOffset = static_cast(row) * stride; const std::size_t dstOffset = static_cast(height - 1 - row) * stride; std::memcpy(flippedPixels.data() + dstOffset, pixels.data() + srcOffset, stride); } glGenTextures(1, &textureId); glBindTexture(GL_TEXTURE_2D, textureId); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, static_cast(width), static_cast(height), 0, GL_BGRA, GL_UNSIGNED_BYTE, flippedPixels.data()); glBindTexture(GL_TEXTURE_2D, 0); if (shouldUninitializeCom) CoUninitialize(); return true; } void OpenGLComposite::bindLayerTextureAssets(const LayerProgram& layerProgram) { const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; const GLuint shaderTextureBase = kSourceHistoryTextureUnitBase + historyCap + historyCap; for (std::size_t index = 0; index < layerProgram.textureBindings.size(); ++index) { glActiveTexture(GL_TEXTURE0 + shaderTextureBase + static_cast(index)); glBindTexture(GL_TEXTURE_2D, layerProgram.textureBindings[index].texture); } glActiveTexture(GL_TEXTURE0); } void OpenGLComposite::destroyDecodeShaderProgram() { if (mDecodeProgram != 0) { glDeleteProgram(mDecodeProgram); mDecodeProgram = 0; } if (mDecodeFragmentShader != 0) { glDeleteShader(mDecodeFragmentShader); mDecodeFragmentShader = 0; } if (mDecodeVertexShader != 0) { glDeleteShader(mDecodeVertexShader); mDecodeVertexShader = 0; } } bool OpenGLComposite::validateTemporalTextureUnitBudget(const std::vector& layerStates, std::string& error) const { const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; unsigned maxAssetTextures = 0; for (const RuntimeRenderState& state : layerStates) { if (state.textureAssets.size() > maxAssetTextures) maxAssetTextures = static_cast(state.textureAssets.size()); } GLint maxTextureUnits = 0; glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureUnits); const unsigned requiredUnits = kSourceHistoryTextureUnitBase + historyCap + historyCap + maxAssetTextures; const unsigned availableUnits = maxTextureUnits > 0 ? static_cast(maxTextureUnits) : 0u; if (requiredUnits > availableUnits) { std::ostringstream message; message << "The current history and shader texture asset configuration requires " << requiredUnits << " fragment texture units, but only " << maxTextureUnits << " are available."; error = message.str(); return false; } return true; } bool OpenGLComposite::createHistoryRing(HistoryRing& ring, unsigned effectiveLength, TemporalHistorySource historySource, std::string& error) { destroyHistoryRing(ring); ring.effectiveLength = effectiveLength; ring.historySource = historySource; if (effectiveLength == 0) return true; ring.slots.resize(effectiveLength); for (HistorySlot& slot : ring.slots) { glGenTextures(1, &slot.texture); glBindTexture(GL_TEXTURE_2D, slot.texture); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mFrameWidth, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glGenFramebuffers(1, &slot.framebuffer); glBindFramebuffer(GL_FRAMEBUFFER, slot.framebuffer); glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, slot.texture, 0); const GLenum framebufferStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER); if (framebufferStatus != GL_FRAMEBUFFER_COMPLETE) { error = "Failed to initialize a temporal history framebuffer."; glBindFramebuffer(GL_FRAMEBUFFER, 0); glBindTexture(GL_TEXTURE_2D, 0); destroyHistoryRing(ring); return false; } } glBindFramebuffer(GL_FRAMEBUFFER, 0); glBindTexture(GL_TEXTURE_2D, 0); return true; } void OpenGLComposite::destroyHistoryRing(HistoryRing& ring) { for (HistorySlot& slot : ring.slots) { if (slot.framebuffer != 0) glDeleteFramebuffers(1, &slot.framebuffer); if (slot.texture != 0) glDeleteTextures(1, &slot.texture); slot.framebuffer = 0; slot.texture = 0; } ring.slots.clear(); ring.nextWriteIndex = 0; ring.filledCount = 0; ring.effectiveLength = 0; ring.historySource = TemporalHistorySource::None; } void OpenGLComposite::destroyTemporalHistoryResources() { destroyHistoryRing(mSourceHistoryRing); for (auto& historyEntry : mPreLayerHistoryByLayerId) destroyHistoryRing(historyEntry.second); mPreLayerHistoryByLayerId.clear(); } void OpenGLComposite::resetTemporalHistoryState() { mSourceHistoryRing.nextWriteIndex = 0; mSourceHistoryRing.filledCount = 0; for (auto& historyEntry : mPreLayerHistoryByLayerId) { historyEntry.second.nextWriteIndex = 0; historyEntry.second.filledCount = 0; } mTemporalHistoryNeedsReset = false; } bool OpenGLComposite::ensureTemporalHistoryResources(const std::vector& layerStates, std::string& error) { const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; const bool sourceHistoryNeeded = std::any_of(layerStates.begin(), layerStates.end(), [](const RuntimeRenderState& state) { return state.isTemporal && state.effectiveTemporalHistoryLength > 0; }); const unsigned sourceHistoryLength = sourceHistoryNeeded ? historyCap : 0; if (mSourceHistoryRing.effectiveLength != sourceHistoryLength) { if (!createHistoryRing(mSourceHistoryRing, sourceHistoryLength, TemporalHistorySource::Source, error)) return false; mTemporalHistoryNeedsReset = true; } std::set requiredPreLayerIds; for (const RuntimeRenderState& state : layerStates) { if (!state.isTemporal || state.temporalHistorySource != TemporalHistorySource::PreLayerInput) continue; requiredPreLayerIds.insert(state.layerId); auto historyIt = mPreLayerHistoryByLayerId.find(state.layerId); if (historyIt == mPreLayerHistoryByLayerId.end() || historyIt->second.effectiveLength != state.effectiveTemporalHistoryLength) { HistoryRing replacement; if (!createHistoryRing(replacement, state.effectiveTemporalHistoryLength, TemporalHistorySource::PreLayerInput, error)) return false; mPreLayerHistoryByLayerId[state.layerId] = std::move(replacement); mTemporalHistoryNeedsReset = true; } } for (auto it = mPreLayerHistoryByLayerId.begin(); it != mPreLayerHistoryByLayerId.end();) { if (requiredPreLayerIds.find(it->first) == requiredPreLayerIds.end()) { destroyHistoryRing(it->second); it = mPreLayerHistoryByLayerId.erase(it); mTemporalHistoryNeedsReset = true; } else { ++it; } } if (mTemporalHistoryNeedsReset) resetTemporalHistoryState(); return true; } void OpenGLComposite::pushFramebufferToHistoryRing(GLuint sourceFramebuffer, HistoryRing& ring) { if (ring.effectiveLength == 0 || ring.slots.empty()) return; HistorySlot& targetSlot = ring.slots[ring.nextWriteIndex]; glBindFramebuffer(GL_READ_FRAMEBUFFER, sourceFramebuffer); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, targetSlot.framebuffer); glBlitFramebuffer(0, 0, mFrameWidth, mFrameHeight, 0, 0, mFrameWidth, mFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); ring.nextWriteIndex = (ring.nextWriteIndex + 1) % ring.slots.size(); ring.filledCount = std::min(ring.filledCount + 1, ring.slots.size()); } GLuint OpenGLComposite::resolveHistoryTexture(const HistoryRing& ring, GLuint fallbackTexture, std::size_t framesAgo) const { if (ring.filledCount == 0 || ring.slots.empty()) return fallbackTexture; const std::size_t clampedOffset = std::min(framesAgo, ring.filledCount - 1); const std::size_t newestIndex = (ring.nextWriteIndex + ring.slots.size() - 1) % ring.slots.size(); const std::size_t slotIndex = (newestIndex + ring.slots.size() - clampedOffset) % ring.slots.size(); return ring.slots[slotIndex].texture != 0 ? ring.slots[slotIndex].texture : fallbackTexture; } unsigned OpenGLComposite::sourceHistoryAvailableCount() const { return static_cast(mSourceHistoryRing.filledCount); } unsigned OpenGLComposite::temporalHistoryAvailableCountForLayer(const std::string& layerId) const { auto it = mPreLayerHistoryByLayerId.find(layerId); if (it == mPreLayerHistoryByLayerId.end()) return 0; return static_cast(it->second.filledCount); } void OpenGLComposite::bindHistorySamplers(const RuntimeRenderState& state, GLuint currentSourceTexture) { const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; for (unsigned index = 0; index < historyCap; ++index) { glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + index); glBindTexture(GL_TEXTURE_2D, resolveHistoryTexture(mSourceHistoryRing, currentSourceTexture, index)); } const GLuint temporalBase = kSourceHistoryTextureUnitBase + historyCap; const HistoryRing* temporalRing = nullptr; auto it = mPreLayerHistoryByLayerId.find(state.layerId); if (it != mPreLayerHistoryByLayerId.end()) temporalRing = &it->second; for (unsigned index = 0; index < historyCap; ++index) { glActiveTexture(GL_TEXTURE0 + temporalBase + index); glBindTexture(GL_TEXTURE_2D, temporalRing ? resolveHistoryTexture(*temporalRing, currentSourceTexture, index) : currentSourceTexture); } glActiveTexture(GL_TEXTURE0); } void OpenGLComposite::renderEffect() { PollRuntimeChanges(); if (mHasNoInputSource) return; if (mFastTransferExtensionAvailable) { // Signal that we're about to draw using mCaptureTexture onto mFBOTexture. VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::CPUtoGPU); } glDisable(GL_BLEND); glDisable(GL_DEPTH_TEST); renderDecodePass(); const std::vector layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mFrameWidth, mFrameHeight) : std::vector(); if (layerStates.empty() || mLayerPrograms.empty()) { glBindFramebuffer(GL_READ_FRAMEBUFFER, mDecodeFrameBuf); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mIdFrameBuf); glBlitFramebuffer(0, 0, mFrameWidth, mFrameHeight, 0, 0, mFrameWidth, mFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf); } else { GLuint sourceTexture = mDecodedTexture; GLuint sourceFrameBuffer = mDecodeFrameBuf; for (std::size_t index = 0; index < layerStates.size() && index < mLayerPrograms.size(); ++index) { const std::size_t remaining = layerStates.size() - index; const bool writeToMain = (remaining % 2) == 1; renderShaderProgram(sourceTexture, writeToMain ? mIdFrameBuf : mLayerTempFrameBuf, mLayerPrograms[index], layerStates[index]); if (layerStates[index].temporalHistorySource == TemporalHistorySource::PreLayerInput) { auto historyIt = mPreLayerHistoryByLayerId.find(layerStates[index].layerId); if (historyIt != mPreLayerHistoryByLayerId.end()) pushFramebufferToHistoryRing(sourceFrameBuffer, historyIt->second); } sourceTexture = writeToMain ? mFBOTexture : mLayerTempTexture; sourceFrameBuffer = writeToMain ? mIdFrameBuf : mLayerTempFrameBuf; } } pushFramebufferToHistoryRing(mDecodeFrameBuf, mSourceHistoryRing); if (mFastTransferExtensionAvailable) VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); } void OpenGLComposite::renderShaderProgram(GLuint sourceTexture, GLuint destinationFrameBuffer, const LayerProgram& layerProgram, const RuntimeRenderState& state) { glBindFramebuffer(GL_FRAMEBUFFER, destinationFrameBuffer); glViewport(0, 0, mFrameWidth, mFrameHeight); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); glActiveTexture(GL_TEXTURE0 + kDecodedVideoTextureUnit); glBindTexture(GL_TEXTURE_2D, sourceTexture); bindHistorySamplers(state, sourceTexture); bindLayerTextureAssets(layerProgram); glBindVertexArray(mFullscreenVAO); glUseProgram(layerProgram.program); updateGlobalParamsBuffer(state, sourceHistoryAvailableCount(), temporalHistoryAvailableCountForLayer(state.layerId)); glDrawArrays(GL_TRIANGLES, 0, 3); glUseProgram(0); glBindVertexArray(0); const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; for (unsigned index = 0; index < historyCap; ++index) { glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + index); glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + historyCap + index); glBindTexture(GL_TEXTURE_2D, 0); } const GLuint shaderTextureBase = kSourceHistoryTextureUnitBase + historyCap + historyCap; for (std::size_t index = 0; index < layerProgram.textureBindings.size(); ++index) { glActiveTexture(GL_TEXTURE0 + shaderTextureBase + static_cast(index)); glBindTexture(GL_TEXTURE_2D, 0); } glActiveTexture(GL_TEXTURE0 + kDecodedVideoTextureUnit); glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0); } void OpenGLComposite::renderDecodePass() { glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf); glViewport(0, 0, mFrameWidth, mFrameHeight); glClear(GL_COLOR_BUFFER_BIT); glActiveTexture(GL_TEXTURE0 + kPackedVideoTextureUnit); glBindTexture(GL_TEXTURE_2D, mCaptureTexture); glBindVertexArray(mFullscreenVAO); glUseProgram(mDecodeProgram); const GLint packedResolutionLocation = glGetUniformLocation(mDecodeProgram, "uPackedVideoResolution"); const GLint decodedResolutionLocation = glGetUniformLocation(mDecodeProgram, "uDecodedVideoResolution"); if (packedResolutionLocation >= 0) glUniform2f(packedResolutionLocation, static_cast(mFrameWidth / 2), static_cast(mFrameHeight)); if (decodedResolutionLocation >= 0) glUniform2f(decodedResolutionLocation, static_cast(mFrameWidth), static_cast(mFrameHeight)); glDrawArrays(GL_TRIANGLES, 0, 3); glUseProgram(0); glBindVertexArray(0); glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0); } bool OpenGLComposite::PollRuntimeChanges() { if (!mRuntimeHost) return true; bool registryChanged = false; bool reloadRequested = false; std::string runtimeError; if (!mRuntimeHost->PollFileChanges(registryChanged, reloadRequested, runtimeError)) { mRuntimeHost->SetCompileStatus(false, runtimeError); broadcastRuntimeState(); return false; } if (registryChanged) broadcastRuntimeState(); if (!reloadRequested) return true; char compilerErrorMessage[1024] = {}; if (!compileLayerPrograms(sizeof(compilerErrorMessage), compilerErrorMessage)) { mRuntimeHost->SetCompileStatus(false, compilerErrorMessage); mRuntimeHost->ClearReloadRequest(); broadcastRuntimeState(); return false; } resetTemporalHistoryState(); broadcastRuntimeState(); return true; } void OpenGLComposite::broadcastRuntimeState() { if (mControlServer) mControlServer->BroadcastState(); } bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength) { std::vector buffer; buffer.reserve(512); AppendStd140Float(buffer, static_cast(state.timeSeconds)); AppendStd140Vec2(buffer, static_cast(state.inputWidth), static_cast(state.inputHeight)); AppendStd140Vec2(buffer, static_cast(state.outputWidth), static_cast(state.outputHeight)); AppendStd140Float(buffer, static_cast(state.frameCount)); AppendStd140Float(buffer, static_cast(state.mixAmount)); AppendStd140Float(buffer, static_cast(state.bypass)); const unsigned effectiveSourceHistoryLength = availableSourceHistoryLength < state.effectiveTemporalHistoryLength ? availableSourceHistoryLength : state.effectiveTemporalHistoryLength; const unsigned effectiveTemporalHistoryLength = (state.temporalHistorySource == TemporalHistorySource::PreLayerInput) ? (availableTemporalHistoryLength < state.effectiveTemporalHistoryLength ? availableTemporalHistoryLength : state.effectiveTemporalHistoryLength) : 0u; AppendStd140Int(buffer, static_cast(effectiveSourceHistoryLength)); AppendStd140Int(buffer, static_cast(effectiveTemporalHistoryLength)); for (const ShaderParameterDefinition& definition : state.parameterDefinitions) { auto valueIt = state.parameterValues.find(definition.id); const ShaderParameterValue value = valueIt != state.parameterValues.end() ? valueIt->second : ShaderParameterValue(); switch (definition.type) { case ShaderParameterType::Float: AppendStd140Float(buffer, value.numberValues.empty() ? 0.0f : static_cast(value.numberValues[0])); break; case ShaderParameterType::Vec2: AppendStd140Vec2(buffer, value.numberValues.size() > 0 ? static_cast(value.numberValues[0]) : 0.0f, value.numberValues.size() > 1 ? static_cast(value.numberValues[1]) : 0.0f); break; case ShaderParameterType::Color: AppendStd140Vec4(buffer, value.numberValues.size() > 0 ? static_cast(value.numberValues[0]) : 1.0f, value.numberValues.size() > 1 ? static_cast(value.numberValues[1]) : 1.0f, value.numberValues.size() > 2 ? static_cast(value.numberValues[2]) : 1.0f, value.numberValues.size() > 3 ? static_cast(value.numberValues[3]) : 1.0f); break; case ShaderParameterType::Boolean: AppendStd140Int(buffer, value.booleanValue ? 1 : 0); break; case ShaderParameterType::Enum: { int selectedIndex = 0; for (std::size_t optionIndex = 0; optionIndex < definition.enumOptions.size(); ++optionIndex) { if (definition.enumOptions[optionIndex].value == value.enumValue) { selectedIndex = static_cast(optionIndex); break; } } AppendStd140Int(buffer, selectedIndex); break; } } } buffer.resize(AlignStd140(buffer.size(), 16), 0); glBindBuffer(GL_UNIFORM_BUFFER, mGlobalParamsUBO); if (mGlobalParamsUBOSize != static_cast(buffer.size())) { glBufferData(GL_UNIFORM_BUFFER, static_cast(buffer.size()), buffer.data(), GL_DYNAMIC_DRAW); mGlobalParamsUBOSize = static_cast(buffer.size()); } else { glBufferSubData(GL_UNIFORM_BUFFER, 0, static_cast(buffer.size()), buffer.data()); } glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mGlobalParamsUBO); glBindBuffer(GL_UNIFORM_BUFFER, 0); return true; } std::string OpenGLComposite::GetRuntimeStateJson() const { return mRuntimeHost ? mRuntimeHost->BuildStateJson() : "{}"; } bool OpenGLComposite::AddLayer(const std::string& shaderId, std::string& error) { if (!mRuntimeHost->AddLayer(shaderId, error)) return false; ReloadShader(); resetTemporalHistoryState(); broadcastRuntimeState(); return true; } bool OpenGLComposite::RemoveLayer(const std::string& layerId, std::string& error) { if (!mRuntimeHost->RemoveLayer(layerId, error)) return false; ReloadShader(); resetTemporalHistoryState(); broadcastRuntimeState(); return true; } bool OpenGLComposite::MoveLayer(const std::string& layerId, int direction, std::string& error) { if (!mRuntimeHost->MoveLayer(layerId, direction, error)) return false; ReloadShader(); resetTemporalHistoryState(); broadcastRuntimeState(); return true; } bool OpenGLComposite::MoveLayerToIndex(const std::string& layerId, std::size_t targetIndex, std::string& error) { if (!mRuntimeHost->MoveLayerToIndex(layerId, targetIndex, error)) return false; ReloadShader(); resetTemporalHistoryState(); broadcastRuntimeState(); return true; } bool OpenGLComposite::SetLayerBypass(const std::string& layerId, bool bypassed, std::string& error) { if (!mRuntimeHost->SetLayerBypass(layerId, bypassed, error)) return false; ReloadShader(); resetTemporalHistoryState(); broadcastRuntimeState(); return true; } bool OpenGLComposite::SetLayerShader(const std::string& layerId, const std::string& shaderId, std::string& error) { if (!mRuntimeHost->SetLayerShader(layerId, shaderId, error)) return false; ReloadShader(); resetTemporalHistoryState(); broadcastRuntimeState(); return true; } bool OpenGLComposite::UpdateLayerParameterJson(const std::string& layerId, const std::string& parameterId, const std::string& valueJson, std::string& error) { JsonValue parsedValue; if (!ParseJson(valueJson, parsedValue, error)) return false; if (!mRuntimeHost->UpdateLayerParameter(layerId, parameterId, parsedValue, error)) return false; broadcastRuntimeState(); return true; } bool OpenGLComposite::UpdateLayerParameterByControlKeyJson(const std::string& layerKey, const std::string& parameterKey, const std::string& valueJson, std::string& error) { JsonValue parsedValue; if (!ParseJson(valueJson, parsedValue, error)) return false; if (!mRuntimeHost->UpdateLayerParameterByControlKey(layerKey, parameterKey, parsedValue, error)) return false; broadcastRuntimeState(); return true; } bool OpenGLComposite::ResetLayerParameters(const std::string& layerId, std::string& error) { if (!mRuntimeHost->ResetLayerParameters(layerId, error)) return false; broadcastRuntimeState(); return true; } bool OpenGLComposite::SaveStackPreset(const std::string& presetName, std::string& error) { if (!mRuntimeHost->SaveStackPreset(presetName, error)) return false; broadcastRuntimeState(); return true; } bool OpenGLComposite::LoadStackPreset(const std::string& presetName, std::string& error) { if (!mRuntimeHost->LoadStackPreset(presetName, error)) return false; ReloadShader(); resetTemporalHistoryState(); broadcastRuntimeState(); return true; } bool OpenGLComposite::CheckOpenGLExtensions() { mFastTransferExtensionAvailable = VideoFrameTransfer::checkFastMemoryTransferAvailable(); if (!mFastTransferExtensionAvailable) OutputDebugStringA("Fast memory transfer extension not available, using regular OpenGL transfer fallback instead\n"); return true; } //////////////////////////////////////////// // PinnedMemoryAllocator //////////////////////////////////////////// // PinnedMemoryAllocator implements the IDeckLinkVideoBufferAllocator interface to be used instead of the // built-in buffer allocator // // For this sample application a custom buffer allocator is used to ensure each address // of buffer memory is aligned on a 4kB boundary required by the OpenGL pinned memory extension. // If the pinned memory extension is not available, this allocator will still be used and // demonstrates how to cache buffer allocations for efficiency. // // The frame cache delays the releasing of buffers until the cache fills up, thereby avoiding an // allocate plus pin operation for every frame, followed by an unpin and deallocate on every frame. PinnedMemoryAllocator::PinnedMemoryAllocator(HDC hdc, HGLRC hglrc, VideoFrameTransfer::Direction direction, unsigned cacheSize, unsigned bufferSize) : mHGLDC(hdc), mHGLRC(hglrc), mRefCount(1), mDirection(direction), mBufferSize(bufferSize), mFrameCacheSize(cacheSize) // large cache size will keep more memory pinned and may result in out of memory errors { } PinnedMemoryAllocator::~PinnedMemoryAllocator() { // Cleanup any unused buffers that remain in the cache while (!mFrameCache.empty()) { unPinAddress(mFrameCache.back()); VirtualFree(mFrameCache.back(), 0, MEM_RELEASE); mFrameCache.pop_back(); } for (auto iter = mFrameTransfer.begin(); iter != mFrameTransfer.end(); ++iter) { delete iter->second; } mFrameTransfer.clear(); } bool PinnedMemoryAllocator::transferFrame(void* address, GLuint gpuTexture) { if (mFrameTransfer.count(address) == 0) { // VideoFrameTransfer prepares and pins address mFrameTransfer[address] = new VideoFrameTransfer(mBufferSize, address, mDirection); } return mFrameTransfer[address]->performFrameTransfer(); } void PinnedMemoryAllocator::waitForTransferComplete(void* address) { if (mFrameTransfer.count(address)) mFrameTransfer[address]->waitForTransferComplete(); } void PinnedMemoryAllocator::unPinAddress(void* address) { // un-pin address only if it has been pinned for transfer if (mFrameTransfer.count(address) > 0) { wglMakeCurrent( mHGLDC, mHGLRC ); mFrameTransfer.erase(address); wglMakeCurrent( NULL, NULL ); } } // IUnknown methods HRESULT STDMETHODCALLTYPE PinnedMemoryAllocator::QueryInterface(REFIID iid, LPVOID* ppv) { if (!ppv) { return E_POINTER; } if (iid == IID_IUnknown || iid == IID_PinnedMemoryAllocator) { *ppv = this; } else if (iid == IID_IDeckLinkVideoBufferAllocator) { *ppv = static_cast(this); } else { *ppv = nullptr; return E_NOINTERFACE; } AddRef(); return S_OK; } ULONG STDMETHODCALLTYPE PinnedMemoryAllocator::AddRef(void) { return ++mRefCount; } ULONG STDMETHODCALLTYPE PinnedMemoryAllocator::Release(void) { int newCount = --mRefCount; if (newCount == 0) delete this; return newCount; } // IDeckLinkMemoryAllocator methods HRESULT STDMETHODCALLTYPE PinnedMemoryAllocator::AllocateVideoBuffer (IDeckLinkVideoBuffer** allocatedBuffer) { std::shared_ptr sharedMemBuffer; // Manage caching of allocated buffers via shared_ptr deleter. auto deleter = [this](void* buffer) mutable { if (mFrameCache.size() < mFrameCacheSize) { mFrameCache.push_back(buffer); } else { // No room left in cache, so un-pin (if it was pinned) and free this buffer unPinAddress(buffer); VirtualFree(buffer, 0, MEM_RELEASE); } // We AddRef this class once the deleter is used because this class owns the mem Release(); }; if (mFrameCache.empty()) { // Allocate memory on a page boundary void* memBuffer = VirtualAlloc(NULL, mBufferSize, MEM_COMMIT | MEM_RESERVE | MEM_WRITE_WATCH, PAGE_READWRITE); if (!memBuffer) return E_OUTOFMEMORY; sharedMemBuffer = std::shared_ptr(memBuffer, deleter); } else { // Re-use most recently released address sharedMemBuffer = std::shared_ptr(mFrameCache.back(), deleter); mFrameCache.pop_back(); } // This class owns the mem so the buffer we return needs to AddRef() this, and Release() in the deleter AddRef(); *allocatedBuffer = new DeckLinkVideoBuffer(sharedMemBuffer, this); return S_OK; } //////////////////////////////////////////// // InputAllocatorPool Class //////////////////////////////////////////// InputAllocatorPool::InputAllocatorPool(HDC hdc, HGLRC hglrc) { mHDC = hdc; mHGLRC = hglrc; } HRESULT InputAllocatorPool::QueryInterface(REFIID iid, void** ppv) { if (!ppv) { return E_POINTER; } if (iid == IID_IUnknown) { *ppv = this; } else if (iid == IID_IDeckLinkVideoBufferAllocatorProvider) { *ppv = static_cast(this); } else { *ppv = nullptr; return E_NOINTERFACE; } AddRef(); return S_OK; } ULONG InputAllocatorPool::AddRef(void) { return ++mRefCount; } ULONG InputAllocatorPool::Release(void) { int newCount = --mRefCount; if (newCount == 0) delete this; return newCount; } HRESULT InputAllocatorPool::GetVideoBufferAllocator( /* [in] */ unsigned int bufferSize, /* [in] */ unsigned int, /* [in] */ unsigned int, /* [in] */ unsigned int, /* [in] */ BMDPixelFormat, /* [out] */ IDeckLinkVideoBufferAllocator **allocator) { if (!allocator) return E_POINTER; auto existing = mAllocatorBySize.find(bufferSize); if (existing != mAllocatorBySize.end()) { *allocator = &*existing->second; (*allocator)->AddRef(); return S_OK; } CComPtr newAllocator; newAllocator.Attach(new (std::nothrow) PinnedMemoryAllocator(mHDC, mHGLRC, VideoFrameTransfer::CPUtoGPU, 3, bufferSize)); if (!newAllocator) return E_OUTOFMEMORY; mAllocatorBySize.emplace(std::make_pair(bufferSize, newAllocator)); *allocator = newAllocator.Detach(); return S_OK; } //////////////////////////////////////////// // DeckLink Video Buffer Class //////////////////////////////////////////// DeckLinkVideoBuffer::DeckLinkVideoBuffer(std::shared_ptr& buffer, PinnedMemoryAllocator* parent) : mParentAllocator(parent), mRefCount(1), mBuffer(buffer) { } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::QueryInterface(REFIID riid, void** ppvObject) { HRESULT result = S_OK; if (ppvObject == nullptr) return E_POINTER; if (riid == IID_IUnknown) { *ppvObject = this; AddRef(); } else if (riid == IID_IDeckLinkVideoBuffer) { *ppvObject = static_cast(this); AddRef(); } else if (riid == IID_PinnedMemoryAllocator) { result = mParentAllocator->QueryInterface(riid, ppvObject); } else { *ppvObject = nullptr; result = E_NOINTERFACE; } return result; } ULONG STDMETHODCALLTYPE DeckLinkVideoBuffer::AddRef() { return ++mRefCount; } ULONG STDMETHODCALLTYPE DeckLinkVideoBuffer::Release() { int newValue = --mRefCount; if (newValue == 0) delete this; return newValue; } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::GetBytes(void** buffer) { if (buffer == nullptr) return E_POINTER; *buffer = mBuffer.get(); return S_OK; } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::GetSize(uint64_t* size) { if (size == nullptr) return E_POINTER; *size = mParentAllocator->bufferSize(); return S_OK; } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::StartAccess(BMDBufferAccessFlags) { return S_OK; } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::EndAccess(BMDBufferAccessFlags) { return S_OK; } //////////////////////////////////////////// // DeckLink Capture Delegate Class //////////////////////////////////////////// CaptureDelegate::CaptureDelegate(OpenGLComposite* pOwner) : m_pOwner(pOwner), mRefCount(1) { } HRESULT CaptureDelegate::QueryInterface(REFIID iid, LPVOID *ppv) { *ppv = NULL; return E_NOINTERFACE; } ULONG CaptureDelegate::AddRef() { return InterlockedIncrement(&mRefCount); } ULONG CaptureDelegate::Release() { int newCount = InterlockedDecrement(&mRefCount); if (newCount == 0) delete this; return newCount; } HRESULT CaptureDelegate::VideoInputFrameArrived(IDeckLinkVideoInputFrame* inputFrame, IDeckLinkAudioInputPacket* /*audioPacket*/) { if (! inputFrame) { // It's possible to receive a NULL inputFrame, but a valid audioPacket. Ignore audio-only frame. return S_OK; } bool hasNoInputSource = (inputFrame->GetFlags() & bmdFrameHasNoInputSource) == bmdFrameHasNoInputSource; m_pOwner->VideoFrameArrived(inputFrame, hasNoInputSource); return S_OK; } HRESULT CaptureDelegate::VideoInputFormatChanged(BMDVideoInputFormatChangedEvents notificationEvents, IDeckLinkDisplayMode *newDisplayMode, BMDDetectedVideoInputFormatFlags detectedSignalFlags) { return S_OK; } //////////////////////////////////////////// // DeckLink Playout Delegate Class //////////////////////////////////////////// PlayoutDelegate::PlayoutDelegate(OpenGLComposite* pOwner) : m_pOwner(pOwner), mRefCount(1) { } HRESULT PlayoutDelegate::QueryInterface(REFIID iid, LPVOID *ppv) { *ppv = NULL; return E_NOINTERFACE; } ULONG PlayoutDelegate::AddRef() { return InterlockedIncrement(&mRefCount); } ULONG PlayoutDelegate::Release() { int newCount = InterlockedDecrement(&mRefCount); if (newCount == 0) delete this; return newCount; } HRESULT PlayoutDelegate::ScheduledFrameCompleted (IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult result) { switch (result) { case bmdOutputFrameDisplayedLate: OutputDebugStringA("ScheduledFrameCompleted() frame did not complete: Frame Displayed Late\n"); break; case bmdOutputFrameDropped: OutputDebugStringA("ScheduledFrameCompleted() frame did not complete: Frame Dropped\n"); break; case bmdOutputFrameCompleted: case bmdOutputFrameFlushed: // Don't log bmdOutputFrameFlushed result since it is expected when Stop() is called break; default: OutputDebugStringA("ScheduledFrameCompleted() frame did not complete: Unknown error\n"); } m_pOwner->PlayoutFrameCompleted(completedFrame, result); return S_OK; } HRESULT PlayoutDelegate::ScheduledPlaybackHasStopped () { return S_OK; }