NV12.hpp
1 #pragma once
2 #include <Gfx/Graph/decoders/ColorSpace.hpp>
3 #include <Gfx/Graph/decoders/GPUVideoDecoder.hpp>
4 extern "C" {
5 #include <libavformat/avformat.h>
6 }
7 
8 namespace score::gfx
9 {
10 
17 {
18  static const constexpr auto nv12_filter_prologue = R"_(#version 450
19 
20 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
21 
22 layout(binding=3) uniform sampler2D y_tex;
23 layout(binding=4) uniform sampler2D uv_tex;
24 
25 layout(location = 0) in vec2 v_texcoord;
26 layout(location = 0) out vec4 fragColor;
27 
28 %2
29 
30 vec4 processTexture(vec4 tex) {
31  vec4 processed = convert_to_rgb(tex);
32  { %1 }
33  return processed;
34 }
35 
36 void main()
37 {
38  float y = texture(y_tex, v_texcoord).r;
39  float u = texture(uv_tex, v_texcoord).r;
40  float v = texture(uv_tex, v_texcoord).a;
41 )_";
42 
43  static const constexpr auto nv12_filter_epilogue = R"_(
44  fragColor = processTexture(vec3(yuv, 1.));
45 })_";
46 
47  Video::ImageFormat& decoder;
48  bool nv21{};
49 
50  NV12Decoder(Video::ImageFormat& d, bool inverted)
51  : decoder{d}
52  , nv21{inverted}
53  {
54  }
55 
56  std::pair<QShader, QShader> init(RenderList& r) override
57  {
58  auto& rhi = *r.state.rhi;
59  const auto w = decoder.width, h = decoder.height;
60 
61  // Y
62  {
63  auto tex = rhi.newTexture(QRhiTexture::R8, {w, h}, 1, QRhiTexture::Flag{});
64  tex->create();
65 
66  auto sampler = rhi.newSampler(
67  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
68  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
69  sampler->create();
70  samplers.push_back({sampler, tex});
71  }
72 
73  // UV
74  {
75  auto tex
76  = rhi.newTexture(QRhiTexture::RGBA8, {w / 4, h / 2}, 1, QRhiTexture::Flag{});
77  tex->create();
78 
79  auto sampler = rhi.newSampler(
80  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
81  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
82  sampler->create();
83  samplers.push_back({sampler, tex});
84  }
85 
86  QString frag = nv12_filter_prologue;
87  if(nv21)
88  frag += " vec3 yuv = vec3(y, v, u);\n";
89  else
90  frag += " vec3 yuv = vec3(y, u, v);\n";
91  frag += nv12_filter_epilogue;
92 
94  r.state, vertexShader(), QString(frag).arg("").arg(colorMatrix(decoder)));
95  }
96 
97  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
98  {
99  setYPixels(res, frame.data[0], frame.linesize[0]);
100  setUVPixels(res, frame.data[1], frame.linesize[1]);
101  }
102 
103  void
104  setYPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
105  {
106  const auto w = decoder.width, h = decoder.height;
107  auto y_tex = samplers[0].texture;
108 
109  QRhiTextureUploadEntry entry{0, 0, createTextureUpload(pixels, w, h, 1, stride)};
110  QRhiTextureUploadDescription desc{entry};
111 
112  res.uploadTexture(y_tex, desc);
113  }
114 
115  void
116  setUVPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
117  {
118  const auto w = decoder.width / 4, h = decoder.height / 2;
119  auto uv_tex = samplers[1].texture;
120 
121  QRhiTextureUploadEntry entry{0, 0, createTextureUpload(pixels, w, h, 4, stride)};
122  QRhiTextureUploadDescription desc{entry};
123 
124  res.uploadTexture(uv_tex, desc);
125  }
126 };
127 
128 }
Processes and renders a video frame on the GPU.
Definition: GPUVideoDecoder.hpp:43
static QRhiTextureSubresourceUploadDescription createTextureUpload(uint8_t *pixels, int w, int h, int bytesPerPixel, int stride)
Utility method to create a QRhiTextureSubresourceUploadDescription.
Definition: GPUVideoDecoder.cpp:22
List of nodes to be rendered to an output.
Definition: RenderList.hpp:19
RenderState & state
RenderState corresponding to this RenderList.
Definition: RenderList.hpp:89
Graphics rendering pipeline for ossia score.
Definition: Filter/PreviewWidget.hpp:12
std::pair< QShader, QShader > makeShaders(const RenderState &v, QString vert, QString frag)
Get a pair of compiled vertex / fragment shaders from GLSL 4.5 sources.
Definition: score-plugin-gfx/Gfx/Graph/Utils.cpp:342
Definition: VideoInterface.hpp:16
Decodes NV12 videos.
Definition: NV12.hpp:17
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: NV12.hpp:56
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: NV12.hpp:97