From d7121ef91af1b15f1038cb4cb51e53bab8c53d89 Mon Sep 17 00:00:00 2001 From: Marcel Ziswiler Date: Mon, 14 Sep 2020 17:24:27 +0200 Subject: qtmultimedia_%.bbappend: apalis-imx8: solve video playback stuttering Solve video playback stuttering issue on Apalis iMX8. Signed-off-by: Marcel Ziswiler --- .../vivante_remap_video_buffer_each_time.patch | 249 +++++++++++++++++++++ qt5-layer/recipes-qt/qt5/qtmultimedia_%.bbappend | 8 + 2 files changed, 257 insertions(+) create mode 100644 qt5-layer/recipes-qt/qt5/qtmultimedia/vivante_remap_video_buffer_each_time.patch create mode 100644 qt5-layer/recipes-qt/qt5/qtmultimedia_%.bbappend diff --git a/qt5-layer/recipes-qt/qt5/qtmultimedia/vivante_remap_video_buffer_each_time.patch b/qt5-layer/recipes-qt/qt5/qtmultimedia/vivante_remap_video_buffer_each_time.patch new file mode 100644 index 0000000..f78ea45 --- /dev/null +++ b/qt5-layer/recipes-qt/qt5/qtmultimedia/vivante_remap_video_buffer_each_time.patch @@ -0,0 +1,249 @@ +From c19c1912399280c081c05b537408108a2dede9ff Mon Sep 17 00:00:00 2001 +From: VaL Doroshchuk +Date: Mon, 4 May 2020 16:50:26 +0200 +Subject: [PATCH 1/1] Vivante: Remap video buffer each time + +Due to some stuttering issues on some platforms, +designed following flow: +1. Some amount of buffers are created to decode the video by a driver. +2. We receive a buffer and generate a texture for it. +3. If a texture is already generated for this memory, just use this texutre. + +It was supposed to solve some performance issues on some platforms. + +But suddenly found that the video buffers are handled/sent not continuously, +also processed asynchronous, which seems leads to the texture/data is updated +while we are using it. + +Sounds like the data should be remapped. + +Suggesting to keep 1 texture available and remove remembering +textures by a buffer. + +Call glTexDirectVIVMap each time when the video buffer is mapped. + +NOTE: Impacts to CPU, the patch increases usage of CPU. + +Pick-to: 5.15 +Fixes: QTBUG-76270 +Change-Id: I15b17f0402a9dccb4414121a1835aee225e4480b +--- + .../videonode/imx6/qsgvivantevideomaterial.cpp | 144 +++++++++------------ + .../videonode/imx6/qsgvivantevideomaterial.h | 5 +- + 2 files changed, 64 insertions(+), 85 deletions(-) + +diff --git a/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp b/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp +index e200e8d16..13963d183 100644 +--- a/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp ++++ b/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp +@@ -60,6 +60,7 @@ QSGVivanteVideoMaterial::QSGVivanteVideoMaterial() : + mWidth(0), + mHeight(0), + mFormat(QVideoFrame::Format_Invalid), ++ mTexture(0), + mCurrentTexture(0), + mMappable(true), + mTexDirectTexture(0) +@@ -71,6 +72,7 @@ QSGVivanteVideoMaterial::QSGVivanteVideoMaterial() : + setFlag(Blending, false); + + mShader = new QSGVivanteVideoMaterialShader; ++ glGenTextures(1, &mTexture); + } + + QSGVivanteVideoMaterial::~QSGVivanteVideoMaterial() +@@ -90,7 +92,7 @@ QSGMaterialShader *QSGVivanteVideoMaterial::createShader() const { + int QSGVivanteVideoMaterial::compare(const QSGMaterial *other) const { + if (this->type() == other->type()) { + const QSGVivanteVideoMaterial *m = static_cast(other); +- if (this->mBitsToTextureMap == m->mBitsToTextureMap) ++ if (this->mTexture == m->mTexture) + return 0; + else + return 1; +@@ -130,15 +132,7 @@ void QSGVivanteVideoMaterial::bind() + + void QSGVivanteVideoMaterial::clearTextures() + { +- for (auto it = mBitsToTextureMap.cbegin(), end = mBitsToTextureMap.cend(); it != end; ++it) { +- GLuint id = it.value(); +-#ifdef QT_VIVANTE_VIDEO_DEBUG +- qDebug() << "delete texture: " << id; +-#endif +- glDeleteTextures(1, &id); +- } +- mBitsToTextureMap.clear(); +- ++ glDeleteTextures(1, &mTexture); + if (mTexDirectTexture) { + glDeleteTextures(1, &mTexDirectTexture); + mTexDirectTexture = 0; +@@ -178,83 +172,71 @@ GLuint QSGVivanteVideoMaterial::vivanteMapping(QVideoFrame vF) + if (vF.map(QAbstractVideoBuffer::ReadOnly)) { + + if (mMappable) { +- if (!mBitsToTextureMap.contains(vF.bits())) { +- // Haven't yet seen this logical address: map to texture. +- GLuint tmpTexId; +- glGenTextures(1, &tmpTexId); +- mBitsToTextureMap.insert(vF.bits(), tmpTexId); +- +- // Determine the full width & height. Full means: actual width/height plus extra padding pixels. +- // The full width can be deduced from the bytesPerLine value. The full height is calculated +- // by calculating the distance between the start of the first and second planes, and dividing +- // it by the stride (= the bytesPerLine). If there is only one plane, we don't worry about +- // extra padding rows, since there are no adjacent extra planes. +- // XXX: This assumes the distance between bits(1) and bits(0) is exactly the size of the first +- // plane (the Y plane in the case of YUV data). A better way would be to have a dedicated +- // planeSize() or planeOffset() getter. +- // Also, this assumes that planes are tightly packed, that is, there is no space between them. +- // It is okay to assume this here though, because the Vivante direct textures also assume that. +- // In other words, if the planes aren't tightly packed, then the direct textures won't be able +- // to render the frame correctly anyway. +- int fullWidth = vF.bytesPerLine() / QSGVivanteVideoNode::getBytesForPixelFormat(vF.pixelFormat()); +- int fullHeight = (vF.planeCount() > 1) ? ((vF.bits(1) - vF.bits(0)) / vF.bytesPerLine()) : vF.height(); +- +- // The uscale is the ratio of actual width to the full width (same for vscale and height). +- // Since the vivante direct textures do not offer a way to explicitly specify the amount of padding +- // columns and rows, we use a trick. We show the full frame - including the padding pixels - in the +- // texture, but render only a subset of that texture. This subset goes from (0,0) to (uScale, vScale). +- // In the shader, the texture coordinates (which go from (0.0, 0.0) to (1.0, 1.0)) are multiplied by +- // the u/v scale values. Since 1.0 * x = x, this effectively limits the texture coordinates from +- // (0.0, 0.0) - (1.0, 1.0) to (0.0, 0.0) - (uScale, vScale). +- float uScale = float(vF.width()) / float(fullWidth); +- float vScale = float(vF.height()) / float(fullHeight); +- mShader->setUVScale(uScale, vScale); +- +- const uchar *constBits = vF.bits(); +- void *bits = (void*)constBits; ++ // Determine the full width & height. Full means: actual width/height plus extra padding pixels. ++ // The full width can be deduced from the bytesPerLine value. The full height is calculated ++ // by calculating the distance between the start of the first and second planes, and dividing ++ // it by the stride (= the bytesPerLine). If there is only one plane, we don't worry about ++ // extra padding rows, since there are no adjacent extra planes. ++ // XXX: This assumes the distance between bits(1) and bits(0) is exactly the size of the first ++ // plane (the Y plane in the case of YUV data). A better way would be to have a dedicated ++ // planeSize() or planeOffset() getter. ++ // Also, this assumes that planes are tightly packed, that is, there is no space between them. ++ // It is okay to assume this here though, because the Vivante direct textures also assume that. ++ // In other words, if the planes aren't tightly packed, then the direct textures won't be able ++ // to render the frame correctly anyway. ++ int fullWidth = vF.bytesPerLine() / QSGVivanteVideoNode::getBytesForPixelFormat(vF.pixelFormat()); ++ int fullHeight = (vF.planeCount() > 1) ? ((vF.bits(1) - vF.bits(0)) / vF.bytesPerLine()) : vF.height(); ++ ++ // The uscale is the ratio of actual width to the full width (same for vscale and height). ++ // Since the vivante direct textures do not offer a way to explicitly specify the amount of padding ++ // columns and rows, we use a trick. We show the full frame - including the padding pixels - in the ++ // texture, but render only a subset of that texture. This subset goes from (0,0) to (uScale, vScale). ++ // In the shader, the texture coordinates (which go from (0.0, 0.0) to (1.0, 1.0)) are multiplied by ++ // the u/v scale values. Since 1.0 * x = x, this effectively limits the texture coordinates from ++ // (0.0, 0.0) - (1.0, 1.0) to (0.0, 0.0) - (uScale, vScale). ++ float uScale = float(vF.width()) / float(fullWidth); ++ float vScale = float(vF.height()) / float(fullHeight); ++ mShader->setUVScale(uScale, vScale); ++ ++ const uchar *constBits = vF.bits(); ++ void *bits = (void*)constBits; + + #ifdef QT_VIVANTE_VIDEO_DEBUG +- qDebug() << Q_FUNC_INFO +- << "new texture, texId: " << tmpTexId +- << "; constBits: " << constBits +- << "; actual/full width: " << vF.width() << "/" << fullWidth +- << "; actual/full height: " << vF.height() << "/" << fullHeight +- << "; UV scale: U " << uScale << " V " << vScale; ++ qDebug() << Q_FUNC_INFO ++ << "new texture, texId: " << mTexture ++ << "; constBits: " << constBits ++ << "; actual/full width: " << vF.width() << "/" << fullWidth ++ << "; actual/full height: " << vF.height() << "/" << fullHeight ++ << "; UV scale: U " << uScale << " V " << vScale; + #endif + +- GLuint physical = ~0U; ++ GLuint physical = ~0U; + #if GST_CHECK_VERSION(1,14,0) +- auto buffer = reinterpret_cast(vF.buffer()); +- auto mem = gst_buffer_peek_memory(buffer->buffer(), 0); +- auto phys_addr = gst_is_phys_memory(mem) ? gst_phys_memory_get_phys_addr(mem) : 0; +- if (phys_addr) +- physical = phys_addr; ++ auto buffer = reinterpret_cast(vF.buffer()); ++ auto mem = gst_buffer_peek_memory(buffer->buffer(), 0); ++ auto phys_addr = gst_is_phys_memory(mem) ? gst_phys_memory_get_phys_addr(mem) : 0; ++ if (phys_addr) ++ physical = phys_addr; + #endif +- glBindTexture(GL_TEXTURE_2D, tmpTexId); +- glTexDirectVIVMap_LOCAL(GL_TEXTURE_2D, +- fullWidth, fullHeight, +- QSGVivanteVideoNode::getVideoFormat2GLFormatMap().value(vF.pixelFormat()), +- &bits, &physical); +- +- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); +- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); +- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); +- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +- glTexDirectInvalidateVIV_LOCAL(GL_TEXTURE_2D); +- +- mMapError = glGetError(); +- if (mMapError == GL_NO_ERROR) +- return tmpTexId; +- +- // Error occurred. +- // Fallback to copying data. +- } else { +- // Fastest path: already seen this logical address. Just +- // indicate that the data belonging to the texture has changed. +- glBindTexture(GL_TEXTURE_2D, mBitsToTextureMap.value(vF.bits())); +- glTexDirectInvalidateVIV_LOCAL(GL_TEXTURE_2D); +- return mBitsToTextureMap.value(vF.bits()); +- } ++ glBindTexture(GL_TEXTURE_2D, mTexture); ++ glTexDirectVIVMap_LOCAL(GL_TEXTURE_2D, ++ fullWidth, fullHeight, ++ QSGVivanteVideoNode::getVideoFormat2GLFormatMap().value(vF.pixelFormat()), ++ &bits, &physical); ++ ++ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); ++ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); ++ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); ++ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); ++ ++ glTexDirectInvalidateVIV_LOCAL(GL_TEXTURE_2D); ++ ++ mMapError = glGetError(); ++ if (mMapError == GL_NO_ERROR) ++ return mTexture; ++ ++ // Error occurred. ++ // Fallback to copying data. + } + + // Cannot map. So copy. +diff --git a/src/plugins/videonode/imx6/qsgvivantevideomaterial.h b/src/plugins/videonode/imx6/qsgvivantevideomaterial.h +index db59e8fc7..bdf3b5b35 100644 +--- a/src/plugins/videonode/imx6/qsgvivantevideomaterial.h ++++ b/src/plugins/videonode/imx6/qsgvivantevideomaterial.h +@@ -40,9 +40,6 @@ + #ifndef QSGVIDEOMATERIAL_VIVMAP_H + #define QSGVIDEOMATERIAL_VIVMAP_H + +-#include +-#include +- + #include + #include + #include +@@ -77,7 +74,7 @@ private: + int mHeight; + QVideoFrame::PixelFormat mFormat; + +- QMap mBitsToTextureMap; ++ GLuint mTexture; + QVideoFrame mCurrentFrame; + GLuint mCurrentTexture; + bool mMappable; +-- +2.16.3 + diff --git a/qt5-layer/recipes-qt/qt5/qtmultimedia_%.bbappend b/qt5-layer/recipes-qt/qt5/qtmultimedia_%.bbappend new file mode 100644 index 0000000..b8d3cfb --- /dev/null +++ b/qt5-layer/recipes-qt/qt5/qtmultimedia_%.bbappend @@ -0,0 +1,8 @@ +FILESEXTRAPATHS_prepend := "${THISDIR}/qtmultimedia/:" + +SRC_URI_append_apalis-imx8 += " \ + file://vivante_remap_video_buffer_each_time.patch \ +" + +#make this machine specific +PACKAGE_ARCH_append_apalis-imx8 = "${MACHINE_ARCH}" -- cgit v1.2.3