NV12.hpp
1 #pragma once
2 #include <Gfx/Graph/decoders/GPUVideoDecoder.hpp>
3 extern "C" {
4 #include <libavformat/avformat.h>
5 }
6 
7 namespace score::gfx
8 {
9 
16 {
17  static const constexpr auto nv12_filter_prologue = R"_(#version 450
18 
19 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
20 
21  layout(binding=3) uniform sampler2D y_tex;
22  layout(binding=4) uniform sampler2D uv_tex;
23 
24  layout(location = 0) in vec2 v_texcoord;
25  layout(location = 0) out vec4 fragColor;
26 
27  const vec3 R_cf = vec3(1.164383, 0.000000, 1.596027);
28  const vec3 G_cf = vec3(1.164383, -0.391762, -0.812968);
29  const vec3 B_cf = vec3(1.164383, 2.017232, 0.000000);
30  const vec3 offset = vec3(-0.0625, -0.5, -0.5);
31 
32  void main ()
33  {
34  float y = texture(y_tex, v_texcoord).r;
35  float u = texture(uv_tex, v_texcoord).r;
36  float v = texture(uv_tex, v_texcoord).a;
37 
38 )_";
39 
40  static const constexpr auto nv12_filter_epilogue = R"_(
41 
42  yuv += offset;
43  fragColor = vec4(0.0, 0.0, 0.0, 1.0);
44  fragColor.r = dot(yuv, R_cf);
45  fragColor.g = dot(yuv, G_cf);
46  fragColor.b = dot(yuv, B_cf);
47  })_";
48 
49  Video::ImageFormat& decoder;
50  bool nv21{};
51 
52  NV12Decoder(Video::ImageFormat& d, bool inverted)
53  : decoder{d}
54  , nv21{inverted}
55  {
56  }
57 
58  std::pair<QShader, QShader> init(RenderList& r) override
59  {
60  auto& rhi = *r.state.rhi;
61  const auto w = decoder.width, h = decoder.height;
62 
63  // Y
64  {
65  auto tex = rhi.newTexture(QRhiTexture::R8, {w, h}, 1, QRhiTexture::Flag{});
66  tex->create();
67 
68  auto sampler = rhi.newSampler(
69  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
70  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
71  sampler->create();
72  samplers.push_back({sampler, tex});
73  }
74 
75  // UV
76  {
77  auto tex
78  = rhi.newTexture(QRhiTexture::RGBA8, {w / 4, h / 2}, 1, QRhiTexture::Flag{});
79  tex->create();
80 
81  auto sampler = rhi.newSampler(
82  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
83  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
84  sampler->create();
85  samplers.push_back({sampler, tex});
86  }
87 
88  QString frag = nv12_filter_prologue;
89  if(nv21)
90  frag += " vec3 yuv = vec3(y, v, u);\n";
91  else
92  frag += " vec3 yuv = vec3(y, u, v);\n";
93  frag += nv12_filter_epilogue;
94 
95  return score::gfx::makeShaders(r.state, vertexShader(), frag);
96  }
97 
98  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
99  {
100  setYPixels(res, frame.data[0], frame.linesize[0]);
101  setUVPixels(res, frame.data[1], frame.linesize[1]);
102  }
103 
104  void
105  setYPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
106  {
107  const auto w = decoder.width, h = decoder.height;
108  auto y_tex = samplers[0].texture;
109 
110  QRhiTextureUploadEntry entry{0, 0, createTextureUpload(pixels, w, h, 1, stride)};
111  QRhiTextureUploadDescription desc{entry};
112 
113  res.uploadTexture(y_tex, desc);
114  }
115 
116  void
117  setUVPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
118  {
119  const auto w = decoder.width / 4, h = decoder.height / 2;
120  auto uv_tex = samplers[1].texture;
121 
122  QRhiTextureUploadEntry entry{0, 0, createTextureUpload(pixels, w, h, 4, stride)};
123  QRhiTextureUploadDescription desc{entry};
124 
125  res.uploadTexture(uv_tex, desc);
126  }
127 };
128 
129 }
Processes and renders a video frame on the GPU.
Definition: GPUVideoDecoder.hpp:43
static QRhiTextureSubresourceUploadDescription createTextureUpload(uint8_t *pixels, int w, int h, int bytesPerPixel, int stride)
Utility method to create a QRhiTextureSubresourceUploadDescription.
Definition: GPUVideoDecoder.cpp:22
List of nodes to be rendered to an output.
Definition: RenderList.hpp:19
RenderState & state
RenderState corresponding to this RenderList.
Definition: RenderList.hpp:89
Graphics rendering pipeline for ossia score.
Definition: Filter/PreviewWidget.hpp:12
std::pair< QShader, QShader > makeShaders(const RenderState &v, QString vert, QString frag)
Get a pair of compiled vertex / fragment shaders from GLSL 4.5 sources.
Definition: score-plugin-gfx/Gfx/Graph/Utils.cpp:342
Definition: VideoInterface.hpp:16
Decodes NV12 videos.
Definition: NV12.hpp:16
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: NV12.hpp:58
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: NV12.hpp:98