/* -LICENSE-START- ** Copyright (c) 2012 Blackmagic Design ** ** Permission is hereby granted, free of charge, to any person or organization ** obtaining a copy of the software and accompanying documentation (the ** "Software") to use, reproduce, display, distribute, sub-license, execute, ** and transmit the Software, and to prepare derivative works of the Software, ** and to permit third-parties to whom the Software is furnished to do so, in ** accordance with: ** ** (1) if the Software is obtained from Blackmagic Design, the End User License ** Agreement for the Software Development Kit ("EULA") available at ** https://www.blackmagicdesign.com/EULA/DeckLinkSDK; or ** ** (2) if the Software is obtained from any third party, such licensing terms ** as notified by that third party, ** ** and all subject to the following: ** ** (3) the copyright notices in the Software and this entire statement, ** including the above license grant, this restriction and the following ** disclaimer, must be included in all copies of the Software, in whole or in ** part, and all derivative works of the Software, unless such copies or ** derivative works are solely in the form of machine-executable object code ** generated by a source language processor. ** ** (4) THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ** OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT ** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE ** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ** DEALINGS IN THE SOFTWARE. ** ** A copy of the Software is available free of charge at ** https://www.blackmagicdesign.com/desktopvideo_sdk under the EULA. ** ** -LICENSE-END- */ #include "OpenGLComposite.h" #include "GLExtensions.h" #include #include #include #include #include #include DEFINE_GUID(IID_PinnedMemoryAllocator, 0xddf921a6, 0x279d, 0x4dcd, 0x86, 0x26, 0x75, 0x7f, 0x58, 0xa8, 0xc4, 0x35); namespace { const char* kSlangShaderRelativePath = "apps/LoopThroughWithOpenGLCompositing/video_effect.slang"; const char* kRuntimeShaderCacheDirectory = "shader_cache"; const char* kRuntimeRawShaderFilename = "video_effect.raw.frag"; const char* kRuntimePatchedShaderFilename = "video_effect.frag"; const char* kVertexShaderSource = "#version 130\n" "out vec2 vTexCoord;\n" "void main()\n" "{\n" " vec2 positions[3] = vec2[3](vec2(-1.0, -1.0), vec2(3.0, -1.0), vec2(-1.0, 3.0));\n" " vec2 texCoords[3] = vec2[3](vec2(0.0, 0.0), vec2(2.0, 0.0), vec2(0.0, 2.0));\n" " gl_Position = vec4(positions[gl_VertexID], 0.0, 1.0);\n" " vTexCoord = texCoords[gl_VertexID];\n" "}\n"; std::string GetExecutableDirectory() { char modulePath[MAX_PATH] = {}; DWORD pathLength = GetModuleFileNameA(NULL, modulePath, MAX_PATH); if (pathLength == 0 || pathLength == MAX_PATH) return std::string(); std::string path(modulePath, pathLength); std::string::size_type slashIndex = path.find_last_of("\\/"); if (slashIndex == std::string::npos) return std::string(); return path.substr(0, slashIndex); } bool ReplaceAll(std::string& text, const std::string& from, const std::string& to) { bool replaced = false; std::string::size_type startPos = 0; while ((startPos = text.find(from, startPos)) != std::string::npos) { text.replace(startPos, from.length(), to); startPos += to.length(); replaced = true; } return replaced; } void CopyErrorMessage(const std::string& message, int errorMessageSize, char* errorMessage) { if (!errorMessage || errorMessageSize <= 0) return; strncpy_s(errorMessage, errorMessageSize, message.c_str(), _TRUNCATE); } bool LoadTextFile(const std::string& path, std::string& contents, std::string& error) { std::ifstream input(path.c_str(), std::ios::binary); if (!input) { error = "Could not open fragment shader file: " + path; return false; } std::ostringstream buffer; buffer << input.rdbuf(); contents = buffer.str(); if (contents.empty()) { error = "Fragment shader file is empty: " + path; return false; } return true; } std::filesystem::path FindRepoRoot() { std::vector rootsToTry; char currentDirBuffer[MAX_PATH] = {}; if (GetCurrentDirectoryA(MAX_PATH, currentDirBuffer) > 0) rootsToTry.push_back(std::filesystem::path(currentDirBuffer)); std::string executableDirectory = GetExecutableDirectory(); if (!executableDirectory.empty()) rootsToTry.push_back(std::filesystem::path(executableDirectory)); for (const std::filesystem::path& startPath : rootsToTry) { std::filesystem::path candidate = startPath; for (int depth = 0; depth < 8 && !candidate.empty(); ++depth) { if (std::filesystem::exists(candidate / kSlangShaderRelativePath)) return candidate; candidate = candidate.parent_path(); } } return std::filesystem::path(); } bool FindSlangCompiler(const std::filesystem::path& repoRoot, std::filesystem::path& slangCompilerPath, std::string& error) { std::filesystem::path thirdPartyPath = repoRoot / "3rdParty"; if (!std::filesystem::exists(thirdPartyPath)) { error = "Could not locate the 3rdParty directory from the application runtime path."; return false; } for (const auto& entry : std::filesystem::directory_iterator(thirdPartyPath)) { if (!entry.is_directory()) continue; std::filesystem::path candidate = entry.path() / "bin" / "slangc.exe"; if (std::filesystem::exists(candidate)) { slangCompilerPath = candidate; return true; } } error = "Could not find slangc.exe under 3rdParty."; return false; } bool RunProcessAndWait(const std::string& commandLine, std::string& error) { STARTUPINFOA startupInfo = {}; PROCESS_INFORMATION processInfo = {}; startupInfo.cb = sizeof(startupInfo); std::vector mutableCommandLine(commandLine.begin(), commandLine.end()); mutableCommandLine.push_back('\0'); if (!CreateProcessA(NULL, mutableCommandLine.data(), NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, NULL, &startupInfo, &processInfo)) { error = "Failed to start slangc.exe."; return false; } WaitForSingleObject(processInfo.hProcess, INFINITE); DWORD exitCode = 0; GetExitCodeProcess(processInfo.hProcess, &exitCode); CloseHandle(processInfo.hThread); CloseHandle(processInfo.hProcess); if (exitCode != 0) { error = "slangc.exe returned a non-zero exit code while compiling the runtime shader."; return false; } return true; } bool PatchGeneratedSlangGLSL(std::string& shaderText, std::string& error) { bool replacedVersion = ReplaceAll(shaderText, "#version 450", "#version 130"); ReplaceAll(shaderText, "#extension GL_EXT_samplerless_texture_functions : require\n", ""); ReplaceAll(shaderText, "layout(row_major) uniform;\n", ""); ReplaceAll(shaderText, "layout(row_major) buffer;\n", ""); ReplaceAll(shaderText, "layout(binding = 0)\nuniform texture2D UYVYtex_0;", "uniform sampler2D UYVYtex;"); ReplaceAll(shaderText, "layout(location = 0)\nout vec4 entryPointParam_fragmentMain_0;\n", ""); ReplaceAll(shaderText, "layout(location = 0)\nin vec2 input_texCoord_0;\n", "in vec2 vTexCoord;\n"); ReplaceAll(shaderText, "UYVYtex_0", "UYVYtex"); ReplaceAll(shaderText, "input_texCoord_0", "vTexCoord"); ReplaceAll(shaderText, "entryPointParam_fragmentMain_0 =", "gl_FragColor ="); if (!replacedVersion) { error = "Generated Slang GLSL did not contain the expected version header."; return false; } return true; } bool BuildFragmentShaderSourceFromSlang(std::string& shaderSource, std::string& error) { std::filesystem::path repoRoot = FindRepoRoot(); if (repoRoot.empty()) { error = "Could not locate the repository root to load video_effect.slang."; return false; } std::filesystem::path slangSourcePath = repoRoot / kSlangShaderRelativePath; if (!std::filesystem::exists(slangSourcePath)) { error = "Could not find video_effect.slang."; return false; } std::filesystem::path slangCompilerPath; if (!FindSlangCompiler(repoRoot, slangCompilerPath, error)) return false; std::filesystem::path shaderCachePath = std::filesystem::path(GetExecutableDirectory()) / kRuntimeShaderCacheDirectory; std::filesystem::create_directories(shaderCachePath); std::filesystem::path rawShaderPath = shaderCachePath / kRuntimeRawShaderFilename; std::filesystem::path patchedShaderPath = shaderCachePath / kRuntimePatchedShaderFilename; std::string commandLine = "\"" + slangCompilerPath.string() + "\" \"" + slangSourcePath.string() + "\" -target glsl -profile glsl_430 -entry fragmentMain -stage fragment -o \"" + rawShaderPath.string() + "\""; if (!RunProcessAndWait(commandLine, error)) return false; if (!LoadTextFile(rawShaderPath.string(), shaderSource, error)) return false; if (!PatchGeneratedSlangGLSL(shaderSource, error)) return false; std::ofstream patchedShaderOutput(patchedShaderPath.string().c_str(), std::ios::binary); if (patchedShaderOutput) patchedShaderOutput << shaderSource; return true; } } OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) : hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC), mCaptureDelegate(NULL), mPlayoutDelegate(NULL), mDLInput(NULL), mDLOutput(NULL), mPlayoutAllocator(NULL), mFrameWidth(0), mFrameHeight(0), mHasNoInputSource(true), mFastTransferExtensionAvailable(false), mCaptureTexture(0), mFBOTexture(0), mProgram(0), mVertexShader(0), mFragmentShader(0), mUYVYtexUniform(-1), mRotateAngle(0.0f), mRotateAngleRate(0.0f) { InitializeCriticalSection(&pMutex); } OpenGLComposite::~OpenGLComposite() { // Cleanup for Capture if (mDLInput != NULL) { mDLInput->SetCallback(NULL); mDLInput->Release(); mDLInput = NULL; } if (mCaptureDelegate != NULL) { mCaptureDelegate->Release(); mCaptureDelegate = NULL; } // Cleanup for Playout while (!mDLOutputVideoFrameQueue.empty()) { IDeckLinkMutableVideoFrame* frameToRelease = mDLOutputVideoFrameQueue.front(); if (frameToRelease != NULL) { frameToRelease->Release(); frameToRelease = NULL; } mDLOutputVideoFrameQueue.pop_front(); } if (mDLOutput != NULL) { mDLOutput->SetScheduledFrameCompletionCallback(NULL); mDLOutput->Release(); mDLOutput = NULL; } if (mPlayoutDelegate != NULL) { mPlayoutDelegate->Release(); mPlayoutDelegate = NULL; } if (mPlayoutAllocator != NULL) { mPlayoutAllocator->Release(); mPlayoutAllocator = NULL; } destroyShaderProgram(); DeleteCriticalSection(&pMutex); } bool OpenGLComposite::InitDeckLink() { bool bSuccess = false; IDeckLinkIterator* pDLIterator = NULL; IDeckLink* pDL = NULL; IDeckLinkProfileAttributes* deckLinkAttributes = NULL; IDeckLinkDisplayModeIterator* pDLDisplayModeIterator = NULL; IDeckLinkDisplayMode* pDLDisplayMode = NULL; BMDDisplayMode displayMode = bmdModeHD1080p5994; // mode to use for capture and playout int outputFrameRowBytes; HRESULT result; result = CoCreateInstance(CLSID_CDeckLinkIterator, NULL, CLSCTX_ALL, IID_IDeckLinkIterator, (void**)&pDLIterator); if (FAILED(result)) { MessageBox(NULL, _T("Please install the Blackmagic DeckLink drivers to use the features of this application."), _T("This application requires the DeckLink drivers installed."), MB_OK); return false; } while (pDLIterator->Next(&pDL) == S_OK) { int64_t duplexMode; if (result = pDL->QueryInterface(IID_IDeckLinkProfileAttributes, (void**)&deckLinkAttributes) != S_OK) { printf("Could not obtain the IDeckLinkProfileAttributes interface - result %08x\n", result); pDL->Release(); pDL = NULL; continue; } result = deckLinkAttributes->GetInt(BMDDeckLinkDuplex, &duplexMode); deckLinkAttributes->Release(); deckLinkAttributes = NULL; if (result != S_OK || duplexMode == bmdDuplexInactive) { pDL->Release(); pDL = NULL; continue; } // Use a full duplex device as capture and playback, or half-duplex device // as capture or playback. bool inputUsed = false; if (!mDLInput && pDL->QueryInterface(IID_IDeckLinkInput, (void**)&mDLInput) == S_OK) inputUsed = true; if (!mDLOutput && (!inputUsed || (duplexMode == bmdDuplexFull))) { if (pDL->QueryInterface(IID_IDeckLinkOutput, (void**)&mDLOutput) != S_OK) mDLOutput = NULL; } pDL->Release(); pDL = NULL; if (mDLOutput && mDLInput) break; } if (! mDLOutput || ! mDLInput) { MessageBox(NULL, _T("Expected both Input and Output DeckLink devices"), _T("This application requires two DeckLink devices."), MB_OK); goto error; } if (mDLOutput->GetDisplayModeIterator(&pDLDisplayModeIterator) != S_OK) { MessageBox(NULL, _T("Cannot get Display Mode Iterator."), _T("DeckLink error."), MB_OK); goto error; } while (pDLDisplayModeIterator->Next(&pDLDisplayMode) == S_OK) { if (pDLDisplayMode->GetDisplayMode() == displayMode) break; pDLDisplayMode->Release(); pDLDisplayMode = NULL; } pDLDisplayModeIterator->Release(); pDLDisplayModeIterator = NULL; if (pDLDisplayMode == NULL) { MessageBox(NULL, _T("Cannot get specified BMDDisplayMode."), _T("DeckLink error."), MB_OK); goto error; } mFrameWidth = pDLDisplayMode->GetWidth(); mFrameHeight = pDLDisplayMode->GetHeight(); if (! CheckOpenGLExtensions()) goto error; if (! InitOpenGLState()) goto error; pDLDisplayMode->GetFrameRate(&mFrameDuration, &mFrameTimescale); // Resize window to match video frame, but scale large formats down by half for viewing if (mFrameWidth < 1920) resizeWindow(mFrameWidth, mFrameHeight); else resizeWindow(mFrameWidth / 2, mFrameHeight / 2); if (mFastTransferExtensionAvailable) { // Initialize fast video frame transfers if (! VideoFrameTransfer::initialize(mFrameWidth, mFrameHeight, mCaptureTexture, mFBOTexture)) { MessageBox(NULL, _T("Cannot initialize video transfers."), _T("VideoFrameTransfer error."), MB_OK); goto error; } } { // Use custom allocators so we pin only once then recycle them CComPtr captureAllocator(new (std::nothrow) InputAllocatorPool(hGLDC, hGLRC)); if (mDLInput->EnableVideoInputWithAllocatorProvider(displayMode, bmdFormat8BitYUV, bmdVideoInputFlagDefault, captureAllocator) != S_OK) goto error; } mCaptureDelegate = new CaptureDelegate(this); if (mDLInput->SetCallback(mCaptureDelegate) != S_OK) goto error; if (mDLOutput->RowBytesForPixelFormat(bmdFormat8BitBGRA, mFrameWidth, &outputFrameRowBytes) != S_OK) goto error; // Use a custom allocator so we pin only once then recycle them mPlayoutAllocator = new PinnedMemoryAllocator(hGLDC, hGLRC, VideoFrameTransfer::GPUtoCPU, 1, outputFrameRowBytes * mFrameHeight); if (mDLOutput->EnableVideoOutput(displayMode, bmdVideoOutputFlagDefault) != S_OK) goto error; // Create a queue of 10 IDeckLinkMutableVideoFrame objects to use for scheduling output video frames. // The ScheduledFrameCompleted() callback will immediately schedule a new frame using the next video frame from this queue. for (int i = 0; i < 10; i++) { // The frame read back from the GPU frame buffer and used for the playout video frame is in BGRA format. // The BGRA frame will be converted on playout to YCbCr either in hardware on most DeckLink cards or in software // within the DeckLink API for DeckLink devices without this hardware conversion. // If you want RGB 4:4:4 format to be played out "over the wire" in SDI, turn on the "Use 4:4:4 SDI" in the control // panel or turn on the bmdDeckLinkConfig444SDIVideoOutput flag using the IDeckLinkConfiguration interface. IDeckLinkMutableVideoFrame* outputFrame; IDeckLinkVideoBuffer* outputFrameBuffer = NULL; if (mPlayoutAllocator->AllocateVideoBuffer(&outputFrameBuffer) != S_OK) goto error; if (mDLOutput->CreateVideoFrameWithBuffer(mFrameWidth, mFrameHeight, outputFrameRowBytes, bmdFormat8BitBGRA, bmdFrameFlagFlipVertical, outputFrameBuffer, &outputFrame) != S_OK) goto error; mDLOutputVideoFrameQueue.push_back(outputFrame); } mPlayoutDelegate = new PlayoutDelegate(this); if (mPlayoutDelegate == NULL) goto error; if (mDLOutput->SetScheduledFrameCompletionCallback(mPlayoutDelegate) != S_OK) goto error; bSuccess = true; error: if (!bSuccess) { if (mDLInput != NULL) { mDLInput->Release(); mDLInput = NULL; } if (mDLOutput != NULL) { mDLOutput->Release(); mDLOutput = NULL; } } if (pDL != NULL) { pDL->Release(); pDL = NULL; } if (pDLDisplayMode != NULL) { pDLDisplayMode->Release(); pDLDisplayMode = NULL; } if (pDLIterator != NULL) { pDLIterator->Release(); pDLIterator = NULL; } return bSuccess; } void OpenGLComposite::paintGL() { // The DeckLink API provides IDeckLinkGLScreenPreviewHelper as a convenient way to view the playout video frames // in a window. However, it performs a copy from host memory to the GPU which is wasteful in this case since // we already have the rendered frame to be played out sitting in the GPU in the mIdFrameBuf frame buffer. // Simply copy the off-screen frame buffer to on-screen frame buffer, scaling to the viewing window size. glBindFramebufferEXT(GL_READ_FRAMEBUFFER, mIdFrameBuf); glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER, 0); glViewport(0, 0, mViewWidth, mViewHeight); glBlitFramebufferEXT(0, 0, mFrameWidth, mFrameHeight, 0, 0, mViewWidth, mViewHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); SwapBuffers(hGLDC); ValidateRect(hGLWnd, NULL); } void OpenGLComposite::resizeGL(WORD width, WORD height) { // We don't set the project or model matrices here since the window data is copied directly from // an off-screen FBO in paintGL(). Just save the width and height for use in paintGL(). mViewWidth = width; mViewHeight = height; } void OpenGLComposite::resizeWindow(int width, int height) { RECT r; if (GetWindowRect(hGLWnd, &r)) { SetWindowPos(hGLWnd, HWND_TOP, r.left, r.top, r.left + width, r.top + height, 0); } } bool OpenGLComposite::InitOpenGLState() { if (! ResolveGLExtensions()) return false; // Prepare the runtime shader program generated from the Slang source file. char compilerErrorMessage[1024]; if (! compileFragmentShader(sizeof(compilerErrorMessage), compilerErrorMessage)) { MessageBoxA(NULL, compilerErrorMessage, "OpenGL shader failed to load or compile", MB_OK); return false; } glClearColor( 0.0f, 0.0f, 0.0f, 0.5f ); // Black background glDisable(GL_DEPTH_TEST); if (! mFastTransferExtensionAvailable) { glGenBuffers(1, &mUnpinnedTextureBuffer); } // Setup the texture which will hold the captured video frame pixels glEnable(GL_TEXTURE_2D); glGenTextures(1, &mCaptureTexture); glBindTexture(GL_TEXTURE_2D, mCaptureTexture); // Parameters to control how texels are sampled from the texture glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP); // Create texture with empty data, we will update it using glTexSubImage2D each frame. // The captured video is YCbCr 4:2:2 packed into a UYVY macropixel. OpenGL has no YCbCr format // so treat it as RGBA 4:4:4:4 by halving the width and using GL_RGBA internal format. glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, mFrameWidth/2, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glBindTexture(GL_TEXTURE_2D, 0); // Create Frame Buffer Object (FBO) to perform off-screen rendering of scene. // This allows the render to be done on a framebuffer with width and height exactly matching the video format. glGenFramebuffersEXT(1, &mIdFrameBuf); glGenRenderbuffersEXT(1, &mIdColorBuf); glGenRenderbuffersEXT(1, &mIdDepthBuf); glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, mIdFrameBuf); // Texture for FBO glGenTextures(1, &mFBOTexture); glBindTexture(GL_TEXTURE_2D, mFBOTexture); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, mFrameWidth, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); // Attach a depth buffer glBindRenderbufferEXT(GL_RENDERBUFFER_EXT, mIdDepthBuf); glRenderbufferStorageEXT(GL_RENDERBUFFER_EXT, GL_DEPTH_COMPONENT, mFrameWidth, mFrameHeight); glFramebufferRenderbufferEXT(GL_FRAMEBUFFER_EXT, GL_DEPTH_ATTACHMENT_EXT, GL_RENDERBUFFER_EXT, mIdDepthBuf); // Attach the texture which stores the playback image glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT, GL_TEXTURE_2D, mFBOTexture, 0); glBindTexture(GL_TEXTURE_2D, 0); glDisable(GL_TEXTURE_2D); GLenum glStatus = glCheckFramebufferStatusEXT(GL_FRAMEBUFFER_EXT); if (glStatus != GL_FRAMEBUFFER_COMPLETE_EXT) { MessageBox(NULL, _T("Cannot initialize framebuffer."), _T("OpenGL initialization error."), MB_OK); return false; } return true; } // // Update the captured video frame texture // void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource) { mHasNoInputSource = hasNoInputSource; if (mHasNoInputSource) return; // don't transfer texture when there's no input long textureSize = inputFrame->GetRowBytes() * inputFrame->GetHeight(); IDeckLinkVideoBuffer* inputFrameBuffer = NULL; void* videoPixels; if (inputFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&inputFrameBuffer) != S_OK) return; if (inputFrameBuffer->StartAccess(bmdBufferAccessRead) != S_OK) { inputFrameBuffer->Release(); return; } inputFrameBuffer->GetBytes(&videoPixels); EnterCriticalSection(&pMutex); wglMakeCurrent( hGLDC, hGLRC ); // make OpenGL context current in this thread if (mFastTransferExtensionAvailable) { CComQIPtr allocator(inputFrameBuffer); if (!allocator || !allocator->transferFrame(videoPixels, mCaptureTexture)) OutputDebugStringA("Capture: transferFrame() failed\n"); allocator->waitForTransferComplete(videoPixels); } else { glEnable(GL_TEXTURE_2D); // Use a straightforward texture buffer glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mUnpinnedTextureBuffer); glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, videoPixels, GL_DYNAMIC_DRAW); glBindTexture(GL_TEXTURE_2D, mCaptureTexture); // NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mFrameWidth/2, mFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glBindTexture(GL_TEXTURE_2D, 0); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); glDisable(GL_TEXTURE_2D); } wglMakeCurrent( NULL, NULL ); LeaveCriticalSection(&pMutex); inputFrameBuffer->EndAccess(bmdBufferAccessRead); inputFrameBuffer->Release(); } // Render the live video texture through the runtime shader into the off-screen framebuffer. // Read the result back from the frame buffer and schedule it for playout. void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult) { EnterCriticalSection(&pMutex); // Get the first frame from the queue IDeckLinkMutableVideoFrame* outputVideoFrame = mDLOutputVideoFrameQueue.front(); mDLOutputVideoFrameQueue.push_back(outputVideoFrame); mDLOutputVideoFrameQueue.pop_front(); // make GL context current in this thread wglMakeCurrent( hGLDC, hGLRC ); // Draw the effect output to the off-screen framebuffer. glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, mIdFrameBuf); renderEffect(); IDeckLinkVideoBuffer* outputVideoFrameBuffer; if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK) { LeaveCriticalSection(&pMutex); return; } if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK) { outputVideoFrameBuffer->Release(); LeaveCriticalSection(&pMutex); return; } void* pFrame; outputVideoFrameBuffer->GetBytes(&pFrame); if (mFastTransferExtensionAvailable) { // Finished with mCaptureTexture VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); if (! mPlayoutAllocator->transferFrame(pFrame, mFBOTexture)) OutputDebugStringA("Playback: transferFrame() failed\n"); paintGL(); // Wait for transfer to system memory to complete mPlayoutAllocator->waitForTransferComplete(pFrame); } else { glReadPixels(0, 0, mFrameWidth, mFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame); paintGL(); } outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite); outputVideoFrameBuffer->Release(); // If the last completed frame was late or dropped, bump the scheduled time further into the future if (completionResult == bmdOutputFrameDisplayedLate || completionResult == bmdOutputFrameDropped) mTotalPlayoutFrames += 2; // Schedule the next frame for playout HRESULT hr = mDLOutput->ScheduleVideoFrame(outputVideoFrame, (mTotalPlayoutFrames * mFrameDuration), mFrameDuration, mFrameTimescale); if (SUCCEEDED(hr)) mTotalPlayoutFrames++; wglMakeCurrent( NULL, NULL ); LeaveCriticalSection(&pMutex); } bool OpenGLComposite::Start() { mTotalPlayoutFrames = 0; // Preroll frames for (unsigned i = 0; i < 5; i++) { // Take each video frame from the front of the queue and move it to the back IDeckLinkMutableVideoFrame* outputVideoFrame = mDLOutputVideoFrameQueue.front(); mDLOutputVideoFrameQueue.push_back(outputVideoFrame); mDLOutputVideoFrameQueue.pop_front(); // Start with a black frame for playout IDeckLinkVideoBuffer* outputVideoFrameBuffer; if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK) return false; if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK) { outputVideoFrameBuffer->Release(); return false; } void* pFrame; outputVideoFrameBuffer->GetBytes((void**)&pFrame); memset(pFrame, 0, outputVideoFrame->GetRowBytes() * mFrameHeight); // 0 is black in RGBA format outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite); outputVideoFrameBuffer->Release(); if (mDLOutput->ScheduleVideoFrame(outputVideoFrame, (mTotalPlayoutFrames * mFrameDuration), mFrameDuration, mFrameTimescale) != S_OK) return false; mTotalPlayoutFrames++; } mDLInput->StartStreams(); mDLOutput->StartScheduledPlayback(0, mFrameTimescale, 1.0); return true; } bool OpenGLComposite::Stop() { mDLInput->StopStreams(); mDLInput->DisableVideoInput(); mDLOutput->StopScheduledPlayback(0, NULL, 0); mDLOutput->DisableVideoOutput(); return true; } bool OpenGLComposite::ReloadShader() { char compilerErrorMessage[1024]; EnterCriticalSection(&pMutex); wglMakeCurrent(hGLDC, hGLRC); bool success = compileFragmentShader(sizeof(compilerErrorMessage), compilerErrorMessage); wglMakeCurrent(NULL, NULL); LeaveCriticalSection(&pMutex); if (!success) MessageBoxA(NULL, compilerErrorMessage, "Slang shader reload failed", MB_OK); return success; } void OpenGLComposite::destroyShaderProgram() { if (mProgram != 0) { glDeleteProgram(mProgram); mProgram = 0; } if (mFragmentShader != 0) { glDeleteShader(mFragmentShader); mFragmentShader = 0; } if (mVertexShader != 0) { glDeleteShader(mVertexShader); mVertexShader = 0; } mUYVYtexUniform = -1; } void OpenGLComposite::renderEffect() { glViewport(0, 0, mFrameWidth, mFrameHeight); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); if (mHasNoInputSource) return; if (mFastTransferExtensionAvailable) { // Signal that we're about to draw using mCaptureTexture onto mFBOTexture. VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::CPUtoGPU); } glDisable(GL_BLEND); glDisable(GL_DEPTH_TEST); glEnable(GL_TEXTURE_2D); glBindTexture(GL_TEXTURE_2D, mCaptureTexture); glUseProgram(mProgram); if (mUYVYtexUniform >= 0) glUniform1i(mUYVYtexUniform, 0); glDrawArrays(GL_TRIANGLES, 0, 3); glUseProgram(0); glBindTexture(GL_TEXTURE_2D, 0); glDisable(GL_TEXTURE_2D); if (mFastTransferExtensionAvailable) VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); } // Compile a fullscreen shader pass from the runtime Slang source. The Slang compiler // emits modern GLSL which we patch into a compatibility-profile shader that can run // inside the sample's WGL context. bool OpenGLComposite::compileFragmentShader(int errorMessageSize, char* errorMessage) { GLsizei errorBufferSize = 0; GLint compileResult = GL_FALSE; GLint linkResult = GL_FALSE; std::string fragmentShaderSource; std::string loadError; const char* vertexSource = kVertexShaderSource; if (!BuildFragmentShaderSourceFromSlang(fragmentShaderSource, loadError)) { CopyErrorMessage(loadError, errorMessageSize, errorMessage); return false; } const char* fragmentSource = fragmentShaderSource.c_str(); GLuint newVertexShader = glCreateShader(GL_VERTEX_SHADER); glShaderSource(newVertexShader, 1, (const GLchar**)&vertexSource, NULL); glCompileShader(newVertexShader); glGetShaderiv(newVertexShader, GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newVertexShader, errorMessageSize, &errorBufferSize, errorMessage); glDeleteShader(newVertexShader); return false; } GLuint newFragmentShader = glCreateShader(GL_FRAGMENT_SHADER); glShaderSource(newFragmentShader, 1, (const GLchar**)&fragmentSource, NULL); glCompileShader(newFragmentShader); glGetShaderiv(newFragmentShader, GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newFragmentShader, errorMessageSize, &errorBufferSize, errorMessage); glDeleteShader(newVertexShader); glDeleteShader(newFragmentShader); return false; } GLuint newProgram = glCreateProgram(); glAttachShader(newProgram, newVertexShader); glAttachShader(newProgram, newFragmentShader); glLinkProgram(newProgram); glGetProgramiv(newProgram, GL_LINK_STATUS, &linkResult); if (linkResult == GL_FALSE) { glGetProgramInfoLog(newProgram, errorMessageSize, &errorBufferSize, errorMessage); glDeleteProgram(newProgram); glDeleteShader(newVertexShader); glDeleteShader(newFragmentShader); return false; } destroyShaderProgram(); mProgram = newProgram; mVertexShader = newVertexShader; mFragmentShader = newFragmentShader; mUYVYtexUniform = glGetUniformLocation(mProgram, "UYVYtex"); return true; } bool OpenGLComposite::CheckOpenGLExtensions() { const GLubyte* strExt; bool hasFBO; // The GL_EXT_framebuffer_object extension is required but GL_AMD_pinned_memory is optional strExt = glGetString (GL_EXTENSIONS); hasFBO = strstr((char*)strExt, "GL_EXT_framebuffer_object") != NULL; mFastTransferExtensionAvailable = VideoFrameTransfer::checkFastMemoryTransferAvailable(); if (!hasFBO) { MessageBox(NULL, _T("Required OpenGL extension \"GL_EXT_framebuffer_object\" is not available."), _T("OpenGL initialization error."), MB_OK); return false; } if (!mFastTransferExtensionAvailable) OutputDebugStringA("Fast memory transfer extension not available, using regular OpenGL transfer fallback instead\n"); return true; } //////////////////////////////////////////// // PinnedMemoryAllocator //////////////////////////////////////////// // PinnedMemoryAllocator implements the IDeckLinkVideoBufferAllocator interface to be used instead of the // built-in buffer allocator // // For this sample application a custom buffer allocator is used to ensure each address // of buffer memory is aligned on a 4kB boundary required by the OpenGL pinned memory extension. // If the pinned memory extension is not available, this allocator will still be used and // demonstrates how to cache buffer allocations for efficiency. // // The frame cache delays the releasing of buffers until the cache fills up, thereby avoiding an // allocate plus pin operation for every frame, followed by an unpin and deallocate on every frame. PinnedMemoryAllocator::PinnedMemoryAllocator(HDC hdc, HGLRC hglrc, VideoFrameTransfer::Direction direction, unsigned cacheSize, unsigned bufferSize) : mHGLDC(hdc), mHGLRC(hglrc), mRefCount(1), mDirection(direction), mBufferSize(bufferSize), mFrameCacheSize(cacheSize) // large cache size will keep more memory pinned and may result in out of memory errors { } PinnedMemoryAllocator::~PinnedMemoryAllocator() { // Cleanup any unused buffers that remain in the cache while (!mFrameCache.empty()) { unPinAddress(mFrameCache.back()); VirtualFree(mFrameCache.back(), 0, MEM_RELEASE); mFrameCache.pop_back(); } for (auto iter = mFrameTransfer.begin(); iter != mFrameTransfer.end(); ++iter) { delete iter->second; } mFrameTransfer.clear(); } bool PinnedMemoryAllocator::transferFrame(void* address, GLuint gpuTexture) { if (mFrameTransfer.count(address) == 0) { // VideoFrameTransfer prepares and pins address mFrameTransfer[address] = new VideoFrameTransfer(mBufferSize, address, mDirection); } return mFrameTransfer[address]->performFrameTransfer(); } void PinnedMemoryAllocator::waitForTransferComplete(void* address) { if (mFrameTransfer.count(address)) mFrameTransfer[address]->waitForTransferComplete(); } void PinnedMemoryAllocator::unPinAddress(void* address) { // un-pin address only if it has been pinned for transfer if (mFrameTransfer.count(address) > 0) { wglMakeCurrent( mHGLDC, mHGLRC ); mFrameTransfer.erase(address); wglMakeCurrent( NULL, NULL ); } } // IUnknown methods HRESULT STDMETHODCALLTYPE PinnedMemoryAllocator::QueryInterface(REFIID iid, LPVOID* ppv) { if (!ppv) { return E_POINTER; } if (iid == IID_IUnknown || iid == IID_PinnedMemoryAllocator) { *ppv = this; } else if (iid == IID_IDeckLinkVideoBufferAllocator) { *ppv = static_cast(this); } else { *ppv = nullptr; return E_NOINTERFACE; } AddRef(); return S_OK; } ULONG STDMETHODCALLTYPE PinnedMemoryAllocator::AddRef(void) { return ++mRefCount; } ULONG STDMETHODCALLTYPE PinnedMemoryAllocator::Release(void) { int newCount = --mRefCount; if (newCount == 0) delete this; return newCount; } // IDeckLinkMemoryAllocator methods HRESULT STDMETHODCALLTYPE PinnedMemoryAllocator::AllocateVideoBuffer (IDeckLinkVideoBuffer** allocatedBuffer) { std::shared_ptr sharedMemBuffer; // Manage caching of allocated buffers via shared_ptr deleter. auto deleter = [this](void* buffer) mutable { if (mFrameCache.size() < mFrameCacheSize) { mFrameCache.push_back(buffer); } else { // No room left in cache, so un-pin (if it was pinned) and free this buffer unPinAddress(buffer); VirtualFree(buffer, 0, MEM_RELEASE); } // We AddRef this class once the deleter is used because this class owns the mem Release(); }; if (mFrameCache.empty()) { // Allocate memory on a page boundary void* memBuffer = VirtualAlloc(NULL, mBufferSize, MEM_COMMIT | MEM_RESERVE | MEM_WRITE_WATCH, PAGE_READWRITE); if (!memBuffer) return E_OUTOFMEMORY; sharedMemBuffer = std::shared_ptr(memBuffer, deleter); } else { // Re-use most recently released address sharedMemBuffer = std::shared_ptr(mFrameCache.back(), deleter); mFrameCache.pop_back(); } // This class owns the mem so the buffer we return needs to AddRef() this, and Release() in the deleter AddRef(); *allocatedBuffer = new DeckLinkVideoBuffer(sharedMemBuffer, this); return S_OK; } //////////////////////////////////////////// // InputAllocatorPool Class //////////////////////////////////////////// InputAllocatorPool::InputAllocatorPool(HDC hdc, HGLRC hglrc) { mHDC = hdc; mHGLRC = hglrc; } HRESULT InputAllocatorPool::QueryInterface(REFIID iid, void** ppv) { if (!ppv) { return E_POINTER; } if (iid == IID_IUnknown) { *ppv = this; } else if (iid == IID_IDeckLinkVideoBufferAllocatorProvider) { *ppv = static_cast(this); } else { *ppv = nullptr; return E_NOINTERFACE; } AddRef(); return S_OK; } ULONG InputAllocatorPool::AddRef(void) { return ++mRefCount; } ULONG InputAllocatorPool::Release(void) { int newCount = --mRefCount; if (newCount == 0) delete this; return newCount; } HRESULT InputAllocatorPool::GetVideoBufferAllocator( /* [in] */ unsigned int bufferSize, /* [in] */ unsigned int, /* [in] */ unsigned int, /* [in] */ unsigned int, /* [in] */ BMDPixelFormat, /* [out] */ IDeckLinkVideoBufferAllocator **allocator) { if (!allocator) return E_POINTER; auto existing = mAllocatorBySize.find(bufferSize); if (existing != mAllocatorBySize.end()) { *allocator = &*existing->second; (*allocator)->AddRef(); return S_OK; } CComPtr newAllocator; newAllocator.Attach(new (std::nothrow) PinnedMemoryAllocator(mHDC, mHGLRC, VideoFrameTransfer::CPUtoGPU, 3, bufferSize)); if (!newAllocator) return E_OUTOFMEMORY; mAllocatorBySize.emplace(std::make_pair(bufferSize, newAllocator)); *allocator = newAllocator.Detach(); return S_OK; } //////////////////////////////////////////// // DeckLink Video Buffer Class //////////////////////////////////////////// DeckLinkVideoBuffer::DeckLinkVideoBuffer(std::shared_ptr& buffer, PinnedMemoryAllocator* parent) : mParentAllocator(parent), mRefCount(1), mBuffer(buffer) { } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::QueryInterface(REFIID riid, void** ppvObject) { HRESULT result = S_OK; if (ppvObject == nullptr) return E_POINTER; if (riid == IID_IUnknown) { *ppvObject = this; AddRef(); } else if (riid == IID_IDeckLinkVideoBuffer) { *ppvObject = static_cast(this); AddRef(); } else if (riid == IID_PinnedMemoryAllocator) { result = mParentAllocator->QueryInterface(riid, ppvObject); } else { *ppvObject = nullptr; result = E_NOINTERFACE; } return result; } ULONG STDMETHODCALLTYPE DeckLinkVideoBuffer::AddRef() { return ++mRefCount; } ULONG STDMETHODCALLTYPE DeckLinkVideoBuffer::Release() { int newValue = --mRefCount; if (newValue == 0) delete this; return newValue; } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::GetBytes(void** buffer) { if (buffer == nullptr) return E_POINTER; *buffer = mBuffer.get(); return S_OK; } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::GetSize(uint64_t* size) { if (size == nullptr) return E_POINTER; *size = mParentAllocator->bufferSize(); return S_OK; } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::StartAccess(BMDBufferAccessFlags) { return S_OK; } HRESULT STDMETHODCALLTYPE DeckLinkVideoBuffer::EndAccess(BMDBufferAccessFlags) { return S_OK; } //////////////////////////////////////////// // DeckLink Capture Delegate Class //////////////////////////////////////////// CaptureDelegate::CaptureDelegate(OpenGLComposite* pOwner) : m_pOwner(pOwner), mRefCount(1) { } HRESULT CaptureDelegate::QueryInterface(REFIID iid, LPVOID *ppv) { *ppv = NULL; return E_NOINTERFACE; } ULONG CaptureDelegate::AddRef() { return InterlockedIncrement(&mRefCount); } ULONG CaptureDelegate::Release() { int newCount = InterlockedDecrement(&mRefCount); if (newCount == 0) delete this; return newCount; } HRESULT CaptureDelegate::VideoInputFrameArrived(IDeckLinkVideoInputFrame* inputFrame, IDeckLinkAudioInputPacket* /*audioPacket*/) { if (! inputFrame) { // It's possible to receive a NULL inputFrame, but a valid audioPacket. Ignore audio-only frame. return S_OK; } bool hasNoInputSource = (inputFrame->GetFlags() & bmdFrameHasNoInputSource) == bmdFrameHasNoInputSource; m_pOwner->VideoFrameArrived(inputFrame, hasNoInputSource); return S_OK; } HRESULT CaptureDelegate::VideoInputFormatChanged(BMDVideoInputFormatChangedEvents notificationEvents, IDeckLinkDisplayMode *newDisplayMode, BMDDetectedVideoInputFormatFlags detectedSignalFlags) { return S_OK; } //////////////////////////////////////////// // DeckLink Playout Delegate Class //////////////////////////////////////////// PlayoutDelegate::PlayoutDelegate(OpenGLComposite* pOwner) : m_pOwner(pOwner), mRefCount(1) { } HRESULT PlayoutDelegate::QueryInterface(REFIID iid, LPVOID *ppv) { *ppv = NULL; return E_NOINTERFACE; } ULONG PlayoutDelegate::AddRef() { return InterlockedIncrement(&mRefCount); } ULONG PlayoutDelegate::Release() { int newCount = InterlockedDecrement(&mRefCount); if (newCount == 0) delete this; return newCount; } HRESULT PlayoutDelegate::ScheduledFrameCompleted (IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult result) { switch (result) { case bmdOutputFrameDisplayedLate: OutputDebugStringA("ScheduledFrameCompleted() frame did not complete: Frame Displayed Late\n"); break; case bmdOutputFrameDropped: OutputDebugStringA("ScheduledFrameCompleted() frame did not complete: Frame Dropped\n"); break; case bmdOutputFrameCompleted: case bmdOutputFrameFlushed: // Don't log bmdOutputFrameFlushed result since it is expected when Stop() is called break; default: OutputDebugStringA("ScheduledFrameCompleted() frame did not complete: Unknown error\n"); } m_pOwner->PlayoutFrameCompleted(completedFrame, result); return S_OK; } HRESULT PlayoutDelegate::ScheduledPlaybackHasStopped () { return S_OK; }