YUV420.hpp
1 #pragma once
2 #include <Gfx/Graph/decoders/ColorSpace.hpp>
3 #include <Gfx/Graph/decoders/GPUVideoDecoder.hpp>
4 
5 extern "C" {
6 #include <libavformat/avformat.h>
7 }
8 
9 namespace score::gfx
10 {
11 
19 {
20  static const constexpr auto frag = R"_(#version 450
21 
22 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
23 
24 layout(binding=3) uniform sampler2D y_tex;
25 layout(binding=4) uniform sampler2D u_tex;
26 layout(binding=5) uniform sampler2D v_tex;
27 
28 layout(location = 0) in vec2 v_texcoord;
29 layout(location = 0) out vec4 fragColor;
30 
31 %2
32 
33 vec4 processTexture(vec4 tex) {
34  vec4 processed = convert_to_rgb(tex);
35  { %1 }
36  return processed;
37 }
38 
39 void main ()
40 {
41  float y = texture(y_tex, v_texcoord).r;
42  float u = texture(u_tex, v_texcoord).r;
43  float v = texture(v_tex, v_texcoord).r;
44 
45  fragColor = processTexture(vec4(y,u,v, 1.));
46 })_";
47 
49  : decoder{d}
50  {
51  }
52 
53  Video::ImageFormat& decoder;
54 
55  std::pair<QShader, QShader> init(RenderList& r) override
56  {
57  auto& rhi = *r.state.rhi;
58  const auto w = decoder.width, h = decoder.height;
59  const auto fmt = QRhiTexture::R8;
60 
61  // Y
62  {
63  auto tex = rhi.newTexture(fmt, {w, h}, 1, QRhiTexture::Flag{});
64  tex->create();
65 
66  auto sampler = rhi.newSampler(
67  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
68  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
69  sampler->create();
70  samplers.push_back({sampler, tex});
71  }
72 
73  // U
74  {
75  auto tex = rhi.newTexture(fmt, {w / 2, h / 2}, 1, QRhiTexture::Flag{});
76  tex->create();
77 
78  auto sampler = rhi.newSampler(
79  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
80  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
81  sampler->create();
82  samplers.push_back({sampler, tex});
83  }
84 
85  // V
86  {
87  auto tex = rhi.newTexture(fmt, {w / 2, h / 2}, 1, QRhiTexture::Flag{});
88  tex->create();
89 
90  auto sampler = rhi.newSampler(
91  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
92  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
93  sampler->create();
94  samplers.push_back({sampler, tex});
95  }
96 
98  r.state, vertexShader(), QString(frag).arg("").arg(colorMatrix(decoder)));
99  }
100 
101  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
102  {
103  setYPixels(res, frame.data[0], frame.linesize[0]);
104  setUPixels(res, frame.data[1], frame.linesize[1]);
105  setVPixels(res, frame.data[2], frame.linesize[2]);
106  }
107 
108  void
109  setYPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
110  {
111  const auto w = decoder.width, h = decoder.height;
112  auto y_tex = samplers[0].texture;
113 
114  QRhiTextureUploadEntry entry{0, 0, createTextureUpload(pixels, w, h, 1, stride)};
115  QRhiTextureUploadDescription desc{entry};
116 
117  res.uploadTexture(y_tex, desc);
118  }
119 
120  void
121  setUPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
122  {
123  const auto w = decoder.width / 2, h = decoder.height / 2;
124  auto u_tex = samplers[1].texture;
125 
126  QRhiTextureUploadEntry entry{0, 0, createTextureUpload(pixels, w, h, 1, stride)};
127  QRhiTextureUploadDescription desc{entry};
128 
129  res.uploadTexture(u_tex, desc);
130  }
131 
132  void
133  setVPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
134  {
135  const auto w = decoder.width / 2, h = decoder.height / 2;
136  auto v_tex = samplers[2].texture;
137 
138  QRhiTextureUploadEntry entry{0, 0, createTextureUpload(pixels, w, h, 1, stride)};
139  QRhiTextureUploadDescription desc{entry};
140  res.uploadTexture(v_tex, desc);
141  }
142 };
143 
144 }
Processes and renders a video frame on the GPU.
Definition: GPUVideoDecoder.hpp:43
static QRhiTextureSubresourceUploadDescription createTextureUpload(uint8_t *pixels, int w, int h, int bytesPerPixel, int stride)
Utility method to create a QRhiTextureSubresourceUploadDescription.
Definition: GPUVideoDecoder.cpp:22
List of nodes to be rendered to an output.
Definition: RenderList.hpp:19
RenderState & state
RenderState corresponding to this RenderList.
Definition: RenderList.hpp:89
Graphics rendering pipeline for ossia score.
Definition: Filter/PreviewWidget.hpp:12
std::pair< QShader, QShader > makeShaders(const RenderState &v, QString vert, QString frag)
Get a pair of compiled vertex / fragment shaders from GLSL 4.5 sources.
Definition: score-plugin-gfx/Gfx/Graph/Utils.cpp:342
Definition: VideoInterface.hpp:16
Decodes YUV420 videos.
Definition: YUV420.hpp:19
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: YUV420.hpp:55
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: YUV420.hpp:101