31#include <Gfx/Graph/decoders/ColorSpace.hpp>
32#include <Gfx/Graph/decoders/GPUVideoDecoder.hpp>
33#include <Gfx/Graph/interop/DrmFourcc.hpp>
34#include <Gfx/Graph/decoders/NV12.hpp>
35#include <Gfx/Graph/decoders/P010.hpp>
36#include <Gfx/Graph/decoders/YUV420.hpp>
37#include <Gfx/Graph/interop/DMABufImport.hpp>
38#include <Gfx/Graph/interop/EglDmaBufImport.hpp>
40#include <Video/VideoInterface.hpp>
42#include <score/gfx/Vulkan.hpp>
47#include <unordered_map>
50#include <libavformat/avformat.h>
51#if __has_include(<libavutil/hwcontext_drm.h>)
52#include <libavutil/hwcontext_drm.h>
53#define SCORE_DRMPRIME_HAS_HWCONTEXT_DRM 1
57#if defined(SCORE_DRMPRIME_HAS_HWCONTEXT_DRM) \
58 && QT_VERSION >= QT_VERSION_CHECK(6, 6, 0)
61#include <QtGui/private/qrhivulkan_p.h>
62#include <vulkan/vulkan.h>
71struct DRMPrimeDecoder : GPUVideoDecoder
81 Backend m_backend{Backend::None};
90 Family m_family{Family::PackedRGB};
91 bool m_is_10bit{
false};
105 struct DmaBufImportKey
107 unsigned long long ino{};
108 unsigned long long dev{};
115 friend bool operator==(
const DmaBufImportKey&,
const DmaBufImportKey&)
118 struct DmaBufImportKeyHash
120 std::size_t operator()(
const DmaBufImportKey& k)
const noexcept
122 std::size_t h = std::hash<unsigned long long>{}(k.ino);
123 auto mix = [&h](std::size_t v) { h ^= v + 0x9e3779b9 + (h << 6) + (h >> 2); };
124 mix(std::hash<unsigned long long>{}(k.dev));
125 mix(std::hash<uint64_t>{}(k.modifier));
126 mix(std::hash<long long>{}(k.offset));
127 mix(std::hash<long long>{}(k.pitch));
128 mix(std::hash<int>{}(k.w));
129 mix(std::hash<int>{}(k.h));
130 mix(std::hash<uint32_t>{}(k.fmt));
137 static std::optional<DmaBufImportKey> importKey(
138 int fd, uint64_t modifier,
long long offset,
long long pitch,
int w,
int h,
139 uint32_t fmt)
noexcept
144 if(fd < 0 || ::fstat(fd, &st) != 0)
147 DmaBufImportKey key{};
150 key.modifier = modifier;
162 static constexpr std::size_t kMaxImports = 32;
166 static constexpr int kNumScratchSlots = 2;
168#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
169 && defined(VK_KHR_external_memory_fd)
170 DMABufPlaneImporter m_vk_importer;
172 std::unordered_map<DmaBufImportKey, DMABufPlaneImporter::PlaneImport,
177 DMABufPlaneImporter::PlaneImport planes[3]{};
179 VkScratch m_vk_scratch[kNumScratchSlots]{};
180 int m_vk_scratchIdx{0};
181 VkFormat m_vk_plane_fmt[3]{};
184 EglDmaBufImporter m_gl_importer;
185 std::unordered_map<DmaBufImportKey, EglDmaBufImporter::PlaneImport,
190 EglDmaBufImporter::PlaneImport planes[3]{};
192 GlScratch m_gl_scratch[kNumScratchSlots]{};
193 int m_gl_scratchIdx{0};
194 unsigned int m_gl_textures[3]{0, 0, 0};
195 uint32_t m_gl_plane_fourcc[3]{};
198 static bool isAvailable(QRhi& rhi)
noexcept
200#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
201 && defined(VK_KHR_external_memory_fd)
202 if(DMABufPlaneImporter::isAvailable(rhi))
205 if(EglDmaBufImporter::isAvailable(rhi))
216 switch(d.hwaccel_sw_format)
218 case AV_PIX_FMT_NV12:
219 m_family = Family::NV12;
222 case AV_PIX_FMT_P010LE:
223 m_family = Family::NV12;
226 case AV_PIX_FMT_YUV420P:
227 m_family = Family::I420;
231 m_family = Family::PackedRGB;
242 void releaseImports() noexcept
244#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
245 && defined(VK_KHR_external_memory_fd)
246 if(m_backend == Backend::Vulkan)
248 const bool scratch_held = std::any_of(
249 std::begin(m_vk_scratch), std::end(m_vk_scratch), [](
const VkScratch& s) {
251 std::begin(s.planes), std::end(s.planes),
252 [](
const DMABufPlaneImporter::PlaneImport& p) {
253 return p.image != VK_NULL_HANDLE;
260 if(m_vk_imports.empty() && !scratch_held)
264 m_vk_importer.waitIdle();
265 for(
auto& [k, pl] : m_vk_imports)
266 m_vk_importer.cleanupPlane(pl);
267 m_vk_imports.clear();
268 for(
auto& slot : m_vk_scratch)
269 for(auto& p : slot.planes)
270 m_vk_importer.cleanupPlane(p);
273 if(m_backend == Backend::OpenGL)
275 for(
auto& [k, pl] : m_gl_imports)
276 m_gl_importer.cleanupPlane(pl);
277 m_gl_imports.clear();
278 for(
auto& slot : m_gl_scratch)
279 for(auto& p : slot.planes)
280 m_gl_importer.cleanupPlane(p);
284 void release(RenderList& r)
override
289 for(
auto& s : samplers)
291 s.texture->destroy();
293 r.state.rhi->finish();
296 GPUVideoDecoder::release(r);
299 ~DRMPrimeDecoder()
override
304 if(m_backend == Backend::OpenGL)
306 if(
auto* ctx = QOpenGLContext::currentContext())
308 if(
auto* funcs = ctx->extraFunctions())
310 for(
unsigned int& tex : m_gl_textures)
312 funcs->glDeleteTextures(1, &tex);
315 for(
unsigned int& tex : m_gl_textures)
320 static constexpr auto packed_frag = R
"_(#version 450
322)_" SCORE_GFX_VIDEO_UNIFORMS R"_(
324 layout(binding=3) uniform sampler2D y_tex;
326 layout(location = 0) in vec2 v_texcoord;
327 layout(location = 0) out vec4 fragColor;
329 vec4 processTexture(vec4 tex) {
330 vec4 processed = tex;
336 fragColor = processTexture(texture(y_tex, v_texcoord));
340 int planeCount() const noexcept
344 case Family::PackedRGB:
return 1;
345 case Family::NV12:
return 2;
346 case Family::I420:
return 3;
351 std::pair<QShader, QShader> init(RenderList& r)
override
353 auto& rhi = *r.state.rhi;
356#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
357 && defined(VK_KHR_external_memory_fd)
358 if(DMABufPlaneImporter::isAvailable(rhi))
360 m_vk_importer.init(rhi);
361 m_backend = Backend::Vulkan;
365 if(EglDmaBufImporter::isAvailable(rhi))
367 if(m_gl_importer.init(rhi))
368 m_backend = Backend::OpenGL;
371 if(m_backend == Backend::None)
374 r.state, vertexShader(), EmptyDecoder::hashtag_no_filter);
378 const auto w = m_decoder.width, h = m_decoder.height;
380 auto makeSamplerTexture = [&](QRhiTexture::Format fmt, QSize size) {
381 auto* tex = rhi.newTexture(fmt, size, 1, QRhiTexture::Flag{});
383 auto* sampler = rhi.newSampler(
384 QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
385 QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
387 samplers.push_back({sampler, tex});
392 case Family::PackedRGB:
397 makeSamplerTexture(QRhiTexture::BGRA8, QSize{w, h});
398#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
399 && defined(VK_KHR_external_memory_fd)
400 m_vk_plane_fmt[0] = VK_FORMAT_UNDEFINED;
408 makeSamplerTexture(QRhiTexture::R16, QSize{w, h});
409 makeSamplerTexture(QRhiTexture::RG16, QSize{w / 2, h / 2});
410#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
411 && defined(VK_KHR_external_memory_fd)
412 m_vk_plane_fmt[0] = VK_FORMAT_R16_UNORM;
413 m_vk_plane_fmt[1] = VK_FORMAT_R16G16_UNORM;
418 m_gl_plane_fourcc[0] = score::gfx::interop::DRM_R16;
419 m_gl_plane_fourcc[1] = score::gfx::interop::DRM_GR1616;
424 makeSamplerTexture(QRhiTexture::R8, QSize{w, h});
425 makeSamplerTexture(QRhiTexture::RG8, QSize{w / 2, h / 2});
426#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
427 && defined(VK_KHR_external_memory_fd)
428 m_vk_plane_fmt[0] = VK_FORMAT_R8_UNORM;
429 m_vk_plane_fmt[1] = VK_FORMAT_R8G8_UNORM;
431 m_gl_plane_fourcc[0] = 0x20203852u;
432 m_gl_plane_fourcc[1] = 0x38385247u;
438 makeSamplerTexture(QRhiTexture::R8, QSize{w, h});
439 makeSamplerTexture(QRhiTexture::R8, QSize{w / 2, h / 2});
440 makeSamplerTexture(QRhiTexture::R8, QSize{w / 2, h / 2});
441#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
442 && defined(VK_KHR_external_memory_fd)
443 m_vk_plane_fmt[0] = VK_FORMAT_R8_UNORM;
444 m_vk_plane_fmt[1] = VK_FORMAT_R8_UNORM;
445 m_vk_plane_fmt[2] = VK_FORMAT_R8_UNORM;
447 m_gl_plane_fourcc[0] = 0x20203852u;
448 m_gl_plane_fourcc[1] = 0x20203852u;
449 m_gl_plane_fourcc[2] = 0x20203852u;
456 if(m_backend == Backend::OpenGL)
458 const int n = planeCount();
459 for(
int i = 0; i < n; ++i)
461 m_gl_textures[i] = score::gfx::createLinearClampGlTexture2D();
463 samplers[i].texture->createFrom(
464 QRhiTexture::NativeTexture{quint64(m_gl_textures[i]), 0});
471 case Family::PackedRGB:
473 r.state, vertexShader(), QString(packed_frag).arg(
""));
479 QString frag = NV12Decoder::nv12_filter_prologue;
481 frag +=
" vec3 yuv = vec3(y * 64.0, u * 64.0, v * 64.0);\n";
483 frag +=
" vec3 yuv = vec3(y, u, v);\n";
484 frag += NV12Decoder::nv12_filter_epilogue;
486 r.state, vertexShader(), frag.arg(
"").arg(colorMatrix(m_decoder)));
491 r.state, vertexShader(),
492 QString(YUV420Decoder::frag).arg(
"").arg(colorMatrix(m_decoder)));
495 r.state, vertexShader(), EmptyDecoder::hashtag_no_filter);
498#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
499 && defined(VK_KHR_external_memory_fd)
502 static QRhiTexture::Format qrhiPackedFmtFromDrmFourcc(uint32_t fourcc)
noexcept
508 return QRhiTexture::BGRA8;
511 return QRhiTexture::RGBA8;
514 return QRhiTexture::RGB10A2;
517 return QRhiTexture::RGBA16F;
519 default:
return QRhiTexture::UnknownFormat;
524 static VkFormat vkPackedFmtFromDrmFourcc(uint32_t fourcc)
noexcept
528 case 0x34325241:
return VK_FORMAT_B8G8R8A8_UNORM;
529 case 0x34325258:
return VK_FORMAT_B8G8R8A8_UNORM;
530 case 0x34324241:
return VK_FORMAT_R8G8B8A8_UNORM;
531 case 0x34324258:
return VK_FORMAT_R8G8B8A8_UNORM;
532 case 0x30335241:
return VK_FORMAT_A2R10G10B10_UNORM_PACK32;
533 case 0x30335258:
return VK_FORMAT_A2R10G10B10_UNORM_PACK32;
534 case 0x30334241:
return VK_FORMAT_A2B10G10R10_UNORM_PACK32;
535 case 0x30334258:
return VK_FORMAT_A2B10G10R10_UNORM_PACK32;
536 case 0x48344241:
return VK_FORMAT_R16G16B16A16_SFLOAT;
537 case 0x48344258:
return VK_FORMAT_R16G16B16A16_SFLOAT;
538 default:
return VK_FORMAT_UNDEFINED;
544 RenderList&, QRhiResourceUpdateBatch& , AVFrame& frame)
override
546 if(m_backend == Backend::None || frame.format != AV_PIX_FMT_DRM_PRIME)
550 =
reinterpret_cast<const AVDRMFrameDescriptor*
>(frame.data[0]);
551 if(!desc || desc->nb_objects < 1 || desc->nb_layers < 1)
563 const int np = planeCount();
567 if(desc->nb_layers == 1 && desc->layers[0].nb_planes >= np)
569 const auto& layer = desc->layers[0];
570 for(
int i = 0; i < np; ++i)
572 const auto& plane = layer.planes[i];
573 p[i].obj_idx = plane.object_index;
574 p[i].offset = plane.offset;
575 p[i].pitch = plane.pitch;
578 else if(desc->nb_layers >= np)
580 for(
int i = 0; i < np; ++i)
582 const auto& plane = desc->layers[i].planes[0];
583 p[i].obj_idx = plane.object_index;
584 p[i].offset = plane.offset;
585 p[i].pitch = plane.pitch;
590 qDebug() <<
"DRMPrimeDecoder: unexpected DRM layout — layers"
591 << desc->nb_layers <<
"but need" << np <<
"planes";
597 p[0].w = m_decoder.width;
598 p[0].h = m_decoder.height;
599 if(m_family == Family::NV12)
601 p[1].w = m_decoder.width / 2;
602 p[1].h = m_decoder.height / 2;
604 else if(m_family == Family::I420)
606 p[1].w = p[2].w = m_decoder.width / 2;
607 p[1].h = p[2].h = m_decoder.height / 2;
610#if QT_HAS_VULKAN && defined(VK_EXT_image_drm_format_modifier) \
611 && defined(VK_KHR_external_memory_fd)
612 if(m_backend == Backend::Vulkan)
616 if(m_family == Family::PackedRGB
617 && m_vk_plane_fmt[0] == VK_FORMAT_UNDEFINED)
619 m_vk_plane_fmt[0] = vkPackedFmtFromDrmFourcc(desc->layers[0].format);
620 if(
const auto qfmt = qrhiPackedFmtFromDrmFourcc(desc->layers[0].format);
621 qfmt != QRhiTexture::UnknownFormat && !samplers.empty()
622 && samplers[0].texture)
625 samplers[0].texture->setFormat(qfmt);
627 if(m_vk_plane_fmt[0] == VK_FORMAT_UNDEFINED)
629 qDebug() <<
"DRMPrimeDecoder: unsupported packed DRM fourcc" << Qt::hex
630 << desc->layers[0].format;
635 for(
int i = 0; i < np; ++i)
637 const auto& obj = desc->objects[p[i].obj_idx];
638 const auto key = importKey(
639 obj.fd, obj.format_modifier, p[i].offset, p[i].pitch, p[i].w, p[i].h,
640 uint32_t(m_vk_plane_fmt[i]));
644 if(
auto it = m_vk_imports.find(*key); it != m_vk_imports.end())
646 samplers[i].texture->createFrom(QRhiTexture::NativeTexture{
647 quint64(it->second.image), VK_IMAGE_LAYOUT_GENERAL});
652 DMABufPlaneImporter::PlaneImport imported{};
653 if(!m_vk_importer.importPlane(
654 imported, obj.fd, obj.format_modifier, p[i].offset, p[i].pitch,
655 m_vk_plane_fmt[i], p[i].w, p[i].h))
657 qDebug() <<
"DRMPrimeDecoder: Vulkan importPlane failed for plane" << i
662 if(key && m_vk_imports.size() < kMaxImports)
664 m_vk_imports.emplace(*key, imported);
670 auto& slot = m_vk_scratch[m_vk_scratchIdx];
671 m_vk_importer.cleanupPlane(slot.planes[i]);
672 slot.planes[i] = imported;
674 m_vk_scratchIdx = (m_vk_scratchIdx + 1) % kNumScratchSlots;
677 samplers[i].texture->createFrom(QRhiTexture::NativeTexture{
678 quint64(imported.image), VK_IMAGE_LAYOUT_GENERAL});
685 if(m_backend == Backend::OpenGL)
689 if(m_family == Family::PackedRGB)
690 m_gl_plane_fourcc[0] = desc->layers[0].format;
692 for(
int i = 0; i < np; ++i)
694 const auto& obj = desc->objects[p[i].obj_idx];
695 const auto key = importKey(
696 obj.fd, obj.format_modifier, p[i].offset, p[i].pitch, p[i].w, p[i].h,
697 m_gl_plane_fourcc[i]);
701 if(
auto it = m_gl_imports.find(*key); it != m_gl_imports.end())
705 if(!m_gl_importer.bindPlane(m_gl_textures[i], it->second))
707 qDebug() <<
"DRMPrimeDecoder: EGL bindPlane failed for plane" << i;
714 EglDmaBufImporter::PlaneImport imported{};
715 if(!m_gl_importer.importPlane(
716 imported, m_gl_textures[i], obj.fd, obj.format_modifier,
717 p[i].offset, p[i].pitch, m_gl_plane_fourcc[i], p[i].w, p[i].h))
719 qDebug() <<
"DRMPrimeDecoder: EGL importPlane failed for plane" << i
720 <<
"fd" << obj.fd <<
"fourcc" << Qt::hex
721 << m_gl_plane_fourcc[i];
725 if(key && m_gl_imports.size() < kMaxImports)
727 m_gl_imports.emplace(*key, imported);
731 auto& slot = m_gl_scratch[m_gl_scratchIdx];
732 m_gl_importer.cleanupPlane(slot.planes[i]);
733 slot.planes[i] = imported;
735 m_gl_scratchIdx = (m_gl_scratchIdx + 1) % kNumScratchSlots;
Graphics rendering pipeline for ossia score.
Definition Filter/PreviewWidget.hpp:11
std::pair< QShader, QShader > makeShaders(const RenderState &v, QString vert, QString frag, int multiViewCount)
Get a pair of compiled vertex / fragment shaders from GLSL 4.5 sources.
Definition score-plugin-gfx/Gfx/Graph/Utils.cpp:1238