#include "InputFrameTexture.h" #include #ifndef GL_FRAMEBUFFER_BINDING #define GL_FRAMEBUFFER_BINDING 0x8CA6 #endif namespace { constexpr GLuint kUyvyTextureUnit = 0; const char* kDecodeVertexShader = R"GLSL( #version 430 core out vec2 vTexCoord; void main() { vec2 positions[3] = vec2[3]( vec2(-1.0, -1.0), vec2( 3.0, -1.0), vec2(-1.0, 3.0)); vec2 texCoords[3] = vec2[3]( vec2(0.0, 0.0), vec2(2.0, 0.0), vec2(0.0, 2.0)); gl_Position = vec4(positions[gl_VertexID], 0.0, 1.0); vTexCoord = texCoords[gl_VertexID]; } )GLSL"; const char* kUyvyDecodeFragmentShader = R"GLSL( #version 430 core layout(binding = 0) uniform sampler2D uPackedUyvy; uniform vec2 uDecodedSize; in vec2 vTexCoord; out vec4 fragColor; vec4 rec709YCbCr2rgba(float yByte, float cbByte, float crByte) { float y = (yByte - 16.0) / 219.0; float cb = (cbByte - 16.0) / 224.0 - 0.5; float cr = (crByte - 16.0) / 224.0 - 0.5; return vec4( y + 1.5748 * cr, y - 0.1873 * cb - 0.4681 * cr, y + 1.8556 * cb, 1.0); } void main() { ivec2 decodedSize = ivec2(uDecodedSize); ivec2 outputCoord = ivec2(clamp(gl_FragCoord.xy, vec2(0.0), vec2(decodedSize - ivec2(1)))); int sourceY = decodedSize.y - 1 - outputCoord.y; ivec2 packedCoord = ivec2(clamp(outputCoord.x / 2, 0, max(decodedSize.x / 2 - 1, 0)), sourceY); vec4 macroPixel = texelFetch(uPackedUyvy, packedCoord, 0) * 255.0; float ySample = (outputCoord.x & 1) != 0 ? macroPixel.a : macroPixel.g; fragColor = clamp(rec709YCbCr2rgba(ySample, macroPixel.r, macroPixel.b), vec4(0.0), vec4(1.0)); } )GLSL"; } InputFrameTexture::~InputFrameTexture() { ShutdownGl(); } GLuint InputFrameTexture::PollAndUpload(InputFrameMailbox* mailbox) { if (mailbox == nullptr) return mTexture; InputFrame frame; if (!mailbox->TryAcquireOldest(frame)) { ++mUploadMisses; mLastUploadMilliseconds = 0.0; return mTexture; } if (frame.bytes != nullptr && frame.pixelFormat == VideoIOPixelFormat::Bgra8 && EnsureTexture(frame)) { mLastFrameFormatSupported = true; const auto uploadStart = std::chrono::steady_clock::now(); UploadBgra8FrameFlippedVertically(frame); const auto uploadEnd = std::chrono::steady_clock::now(); mLastUploadMilliseconds = std::chrono::duration_cast>(uploadEnd - uploadStart).count(); ++mUploadedFrames; } else if (frame.bytes != nullptr && frame.pixelFormat == VideoIOPixelFormat::Uyvy8 && EnsureTexture(frame) && EnsureRawUyvyTexture(frame) && EnsureDecodeProgram()) { mLastFrameFormatSupported = true; const auto uploadStart = std::chrono::steady_clock::now(); UploadUyvy8Frame(frame); DecodeUyvy8Frame(frame); const auto uploadEnd = std::chrono::steady_clock::now(); mLastUploadMilliseconds = std::chrono::duration_cast>(uploadEnd - uploadStart).count(); ++mUploadedFrames; } else { mLastFrameFormatSupported = frame.pixelFormat == VideoIOPixelFormat::Bgra8 || frame.pixelFormat == VideoIOPixelFormat::Uyvy8; mLastUploadMilliseconds = 0.0; } mailbox->Release(frame); return mTexture; } void InputFrameTexture::ShutdownGl() { if (mTexture != 0) glDeleteTextures(1, &mTexture); if (mRawTexture != 0) glDeleteTextures(1, &mRawTexture); mTexture = 0; mRawTexture = 0; mWidth = 0; mHeight = 0; mRawWidth = 0; mRawHeight = 0; DestroyDecodeResources(); } bool InputFrameTexture::EnsureTexture(const InputFrame& frame) { if (frame.width == 0 || frame.height == 0) return false; if (mTexture != 0 && mWidth == frame.width && mHeight == frame.height) return true; ShutdownGl(); glGenTextures(1, &mTexture); glBindTexture(GL_TEXTURE_2D, mTexture); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D( GL_TEXTURE_2D, 0, GL_RGBA8, static_cast(frame.width), static_cast(frame.height), 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, nullptr); glBindTexture(GL_TEXTURE_2D, 0); mWidth = frame.width; mHeight = frame.height; return mTexture != 0; } bool InputFrameTexture::EnsureRawUyvyTexture(const InputFrame& frame) { if (frame.width == 0 || frame.height == 0) return false; const unsigned rawWidth = (frame.width + 1u) / 2u; if (mRawTexture != 0 && mRawWidth == rawWidth && mRawHeight == frame.height) return true; if (mRawTexture != 0) glDeleteTextures(1, &mRawTexture); mRawTexture = 0; glGenTextures(1, &mRawTexture); glBindTexture(GL_TEXTURE_2D, mRawTexture); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D( GL_TEXTURE_2D, 0, GL_RGBA8, static_cast(rawWidth), static_cast(frame.height), 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); glBindTexture(GL_TEXTURE_2D, 0); mRawWidth = rawWidth; mRawHeight = frame.height; return mRawTexture != 0; } void InputFrameTexture::UploadBgra8FrameFlippedVertically(const InputFrame& frame) { glBindTexture(GL_TEXTURE_2D, mTexture); glPixelStorei(GL_UNPACK_ALIGNMENT, 4); glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.rowBytes > 0 ? static_cast(frame.rowBytes / 4) : 0); const unsigned char* sourceBytes = static_cast(frame.bytes); for (unsigned destinationY = 0; destinationY < frame.height; ++destinationY) { const unsigned sourceY = frame.height - 1u - destinationY; const unsigned char* sourceRow = sourceBytes + static_cast(sourceY) * static_cast(frame.rowBytes); glTexSubImage2D( GL_TEXTURE_2D, 0, 0, static_cast(destinationY), static_cast(frame.width), 1, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, sourceRow); } glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); glBindTexture(GL_TEXTURE_2D, 0); } void InputFrameTexture::UploadUyvy8Frame(const InputFrame& frame) { glBindTexture(GL_TEXTURE_2D, mRawTexture); glPixelStorei(GL_UNPACK_ALIGNMENT, 4); glPixelStorei(GL_UNPACK_ROW_LENGTH, frame.rowBytes > 0 ? static_cast(frame.rowBytes / 4) : 0); glTexSubImage2D( GL_TEXTURE_2D, 0, 0, 0, static_cast((frame.width + 1u) / 2u), static_cast(frame.height), GL_RGBA, GL_UNSIGNED_BYTE, frame.bytes); glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); glBindTexture(GL_TEXTURE_2D, 0); } void InputFrameTexture::DecodeUyvy8Frame(const InputFrame& frame) { GLint previousFramebuffer = 0; glGetIntegerv(GL_FRAMEBUFFER_BINDING, &previousFramebuffer); if (mDecodeFramebuffer == 0) glGenFramebuffers(1, &mDecodeFramebuffer); glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFramebuffer); glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mTexture, 0); if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) { glBindFramebuffer(GL_FRAMEBUFFER, static_cast(previousFramebuffer)); return; } glViewport(0, 0, static_cast(frame.width), static_cast(frame.height)); glDisable(GL_SCISSOR_TEST); glDisable(GL_DEPTH_TEST); glDisable(GL_BLEND); glUseProgram(mDecodeProgram); const GLint decodedSizeLocation = glGetUniformLocation(mDecodeProgram, "uDecodedSize"); if (decodedSizeLocation >= 0) glUniform2f(decodedSizeLocation, static_cast(frame.width), static_cast(frame.height)); glActiveTexture(GL_TEXTURE0 + kUyvyTextureUnit); glBindTexture(GL_TEXTURE_2D, mRawTexture); glBindVertexArray(mDecodeVertexArray); glDrawArrays(GL_TRIANGLES, 0, 3); glBindVertexArray(0); glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0); glUseProgram(0); glBindFramebuffer(GL_FRAMEBUFFER, static_cast(previousFramebuffer)); } bool InputFrameTexture::EnsureDecodeProgram() { if (mDecodeProgram != 0) return true; if (!CompileShader(GL_VERTEX_SHADER, kDecodeVertexShader, mDecodeVertexShader)) return false; if (!CompileShader(GL_FRAGMENT_SHADER, kUyvyDecodeFragmentShader, mDecodeFragmentShader)) return false; if (!LinkProgram(mDecodeVertexShader, mDecodeFragmentShader, mDecodeProgram)) return false; glUseProgram(mDecodeProgram); const GLint samplerLocation = glGetUniformLocation(mDecodeProgram, "uPackedUyvy"); if (samplerLocation >= 0) glUniform1i(samplerLocation, static_cast(kUyvyTextureUnit)); glUseProgram(0); if (mDecodeVertexArray == 0) glGenVertexArrays(1, &mDecodeVertexArray); return mDecodeProgram != 0 && mDecodeVertexArray != 0; } void InputFrameTexture::DestroyDecodeResources() { if (mDecodeFramebuffer != 0) glDeleteFramebuffers(1, &mDecodeFramebuffer); if (mDecodeVertexArray != 0) glDeleteVertexArrays(1, &mDecodeVertexArray); if (mDecodeProgram != 0) glDeleteProgram(mDecodeProgram); if (mDecodeVertexShader != 0) glDeleteShader(mDecodeVertexShader); if (mDecodeFragmentShader != 0) glDeleteShader(mDecodeFragmentShader); mDecodeFramebuffer = 0; mDecodeVertexArray = 0; mDecodeProgram = 0; mDecodeVertexShader = 0; mDecodeFragmentShader = 0; } bool InputFrameTexture::CompileShader(GLenum shaderType, const char* source, GLuint& shader) { shader = glCreateShader(shaderType); glShaderSource(shader, 1, &source, nullptr); glCompileShader(shader); GLint compileResult = GL_FALSE; glGetShaderiv(shader, GL_COMPILE_STATUS, &compileResult); if (compileResult != GL_FALSE) return true; glDeleteShader(shader); shader = 0; return false; } bool InputFrameTexture::LinkProgram(GLuint vertexShader, GLuint fragmentShader, GLuint& program) { program = glCreateProgram(); glAttachShader(program, vertexShader); glAttachShader(program, fragmentShader); glLinkProgram(program); GLint linkResult = GL_FALSE; glGetProgramiv(program, GL_LINK_STATUS, &linkResult); if (linkResult != GL_FALSE) return true; glDeleteProgram(program); program = 0; return false; }