RGBA.hpp
1 #pragma once
2 #include <Gfx/Graph/decoders/GPUVideoDecoder.hpp>
3 extern "C" {
4 #include <libavformat/avformat.h>
5 }
6 
7 namespace score::gfx
8 {
10 {
11  static const constexpr auto rgb_filter = R"_(#version 450
12 
13 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
14 
15  layout(binding=3) uniform sampler2D y_tex;
16 
17  layout(location = 0) in vec2 v_texcoord;
18  layout(location = 0) out vec4 fragColor;
19 
20  vec4 processTexture(vec4 tex) {
21  vec4 processed = tex;
22  { %1 }
23  return processed;
24  }
25 
26  void main ()
27  {
28  fragColor = processTexture(texture(y_tex, v_texcoord));
29  })_";
30 
32  QRhiTexture::Format fmt, int bytes_per_pixel, Video::ImageFormat& d,
33  QString f = "")
34  : format{fmt}
35  , bytes_per_pixel{bytes_per_pixel}
36  , decoder{d}
37  , filter{std::move(f)}
38  {
39  }
40  QRhiTexture::Format format;
41  int bytes_per_pixel{}; // bpp/8 !
42  Video::ImageFormat& decoder;
43  QString filter;
44 
45  std::pair<QShader, QShader> init(RenderList& r) override
46  {
47  auto& rhi = *r.state.rhi;
48  const auto w = decoder.width, h = decoder.height;
49 
50  {
51  // Create a texture
52  auto tex = rhi.newTexture(format, QSize{w, h}, 1, QRhiTexture::Flag{});
53  tex->create();
54 
55  // Create a sampler
56  auto sampler = rhi.newSampler(
57  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
58  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
59  sampler->create();
60 
61  // Store both
62  samplers.push_back({sampler, tex});
63  }
64 
66  r.state, vertexShader(), QString(rgb_filter).arg(filter));
67  }
68 
69  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
70  {
71  // Nothing particular, we just upload the whole buffer
72  setPixels(res, frame.data[0], frame.linesize[0]);
73  }
74 
75  void
76  setPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
77  {
78  const auto w = decoder.width, h = decoder.height;
79  auto y_tex = samplers[0].texture;
80 
81  QRhiTextureUploadEntry entry{
82  0, 0, createTextureUpload(pixels, w, h, bytes_per_pixel, stride)};
83 
84  QRhiTextureUploadDescription desc{entry};
85  res.uploadTexture(y_tex, desc);
86  }
87 };
88 
90 {
91  static const constexpr auto rgb_filter = R"_(#version 450
92 
93 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
94 
95  %1
96  layout(location = 0) in vec2 v_texcoord;
97  layout(location = 0) out vec4 fragColor;
98 
99  vec4 processTexture(vec4 tex) {
100  vec4 processed = tex;
101  { %2 }
102  return processed;
103  }
104 
105  void main ()
106  {
107  vec4 tex = vec4(1.);
108  %3
109  fragColor = processTexture(tex);
110  })_";
111 
113  QRhiTexture::Format fmt, int bytes_per_pixel, QString planes,
114  Video::ImageFormat& d, QString f = "")
115  : format{fmt}
116  , bytes_per_pixel{bytes_per_pixel}
117  , planes{planes}
118  , decoder{d}
119  , filter{std::move(f)}
120  {
121  }
122  QRhiTexture::Format format;
123  int bytes_per_pixel{}; // bpp/8 !
124  QString planes{};
125  Video::ImageFormat& decoder;
126  QString filter;
127 
128  std::pair<QShader, QShader> init(RenderList& r) override
129  {
130  auto& rhi = *r.state.rhi;
131  const auto w = decoder.width, h = decoder.height;
132 
133  QString samplers_code;
134  QString read_texture_code;
135 
136  const int binding_orig = 3;
137  for(int i = 0; i < planes.size(); i++)
138  {
139  samplers_code += QString(
140  " layout(binding=%1) uniform sampler2D tconst mat4 "
141  "colorspace_matrix = %2;;\n")
142  .arg(binding_orig + i)
143  .arg(i);
144  read_texture_code += QString(
145  " tex.%1 = texture(tconst mat4 colorspace_matrix = "
146  "%2;, v_texcoord).r;\n")
147  .arg(planes[i])
148  .arg(i);
149 
150  // Create a texture
151  auto tex = rhi.newTexture(format, QSize{w, h}, 1, QRhiTexture::Flag{});
152  tex->create();
153 
154  // Create a sampler
155  auto sampler = rhi.newSampler(
156  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
157  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
158  sampler->create();
159 
160  // Store both
161  samplers.push_back({sampler, tex});
162  }
163 
165  r.state, vertexShader(),
166  QString(rgb_filter).arg(samplers_code).arg(filter).arg(read_texture_code));
167  }
168 
169  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
170  {
171  // Nothing particular, we just upload the whole buffer
172  for(int i = 0; i < planes.size(); i++)
173  {
174  setPixels(res, samplers[i].texture, frame.data[i], frame.linesize[i]);
175  }
176  }
177 
178  void setPixels(
179  QRhiResourceUpdateBatch& res, QRhiTexture* tex, uint8_t* pixels,
180  int stride) const noexcept
181  {
182  const auto w = decoder.width, h = decoder.height;
183 
184  QRhiTextureUploadEntry entry{
185  0, 0, createTextureUpload(pixels, w, h, bytes_per_pixel, stride)};
186 
187  QRhiTextureUploadDescription desc{entry};
188  res.uploadTexture(tex, desc);
189  }
190 };
191 
193 {
194  static const constexpr auto rgb_filter = R"_(#version 450
195 
196 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
197 
198  layout(binding=3) uniform sampler2DRect y_tex;
199 
200  layout(location = 0) in vec2 v_texcoord;
201  layout(location = 0) out vec4 fragColor;
202 
203  vec4 processTexture(vec4 tex) {
204  vec4 processed = tex;
205  { %1 }
206  return processed;
207  }
208 
209  void main ()
210  {
211  fragColor = processTexture(texture(y_tex, v_texcoord));
212  })_";
213 
214  static constexpr const char* vertex = R"_(#version 450
215 layout(location = 0) in vec2 position;
216 layout(location = 1) in vec2 texcoord;
217 
218 layout(location = 0) out vec2 v_texcoord;
219 
220 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
221 
222 out gl_PerVertex { vec4 gl_Position; };
223 
224 void main()
225 {
226  v_texcoord = texcoord * mat.texSz.xy;
227  gl_Position = renderer.clipSpaceCorrMatrix * vec4(position.x * mat.scale.x, position.y * mat.scale.y, 0.0, 1.);
228 }
229 )_";
230 
232  QRhiTexture::Format fmt, int bytes_per_pixel, Video::ImageFormat& d,
233  QString f = "")
234  : format{fmt}
235  , bytes_per_pixel{bytes_per_pixel}
236  , decoder{d}
237  , filter{std::move(f)}
238  {
239  }
240  QRhiTexture::Format format;
241  int bytes_per_pixel{}; // bpp/8 !
242  Video::ImageFormat& decoder;
243  QString filter;
244 
245  std::pair<QShader, QShader> init(RenderList& r) override
246  {
247  auto& rhi = *r.state.rhi;
248  const auto w = decoder.width, h = decoder.height;
249 
250  {
251  // Create a texture
252  auto tex = rhi.newTexture(format, QSize{w, h}, 1, QRhiTexture::Flag{});
253  tex->create();
254 
255  // Create a sampler
256  auto sampler = rhi.newSampler(
257  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
258  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
259  sampler->create();
260 
261  // Store both
262  samplers.push_back({sampler, tex});
263  }
264 
265  return score::gfx::makeShaders(r.state, vertex, QString(rgb_filter).arg(filter));
266  }
267 
268  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
269  {
270  // Nothing particular, we just upload the whole buffer
271  setPixels(res, frame.data[0], frame.linesize[0]);
272  }
273 
274  void
275  setPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
276  {
277  const auto w = decoder.width, h = decoder.height;
278  auto y_tex = samplers[0].texture;
279 
280  QRhiTextureUploadEntry entry{
281  0, 0, createTextureUpload(pixels, w, h, bytes_per_pixel, stride)};
282 
283  QRhiTextureUploadDescription desc{entry};
284  res.uploadTexture(y_tex, desc);
285  }
286 };
287 
289 {
290  static const constexpr auto rgb_filter = R"_(#version 450
291 
292 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
293 
294  layout(binding=3) uniform sampler2D y_tex;
295 
296  layout(location = 0) in vec2 v_texcoord;
297  layout(location = 0) out vec4 fragColor;
298 
299  vec4 processTexture(vec4 tex) {
300  vec4 processed = tex;
301  { %1 }
302  return processed;
303  }
304 
305  void main ()
306  {
307  float w = mat.texSz.x;
308  float h = mat.texSz.y;
309  int x = int(floor(v_texcoord.x * w) * 3.);
310  int y = int(v_texcoord.y * h);
311  float r = texelFetch(y_tex, ivec2(x + 0, y), 0).r;
312  float g = texelFetch(y_tex, ivec2(x + 1, y), 0).r;
313  float b = texelFetch(y_tex, ivec2(x + 2, y), 0).r;
314  fragColor = processTexture(vec4(r, g, b, 1.));
315  })_";
316 
317  RGB24Decoder(Video::ImageFormat& d, QString f = "")
318  : bytes_per_pixel{3}
319  , decoder{d}
320  , filter{std::move(f)}
321  {
322  }
323  QRhiTexture::Format format;
324  int bytes_per_pixel{}; // bpp/8 !
325  Video::ImageFormat& decoder;
326  QString filter;
327 
328  std::pair<QShader, QShader> init(RenderList& r) override
329  {
330  auto& rhi = *r.state.rhi;
331  const auto w = decoder.width, h = decoder.height;
332 
333  {
334  // Create a texture
335  auto tex = rhi.newTexture(QRhiTexture::R8, QSize{w * 3, h}, 1, QRhiTexture::sRGB);
336  tex->create();
337 
338  // Create a sampler
339  auto sampler = rhi.newSampler(
340  QRhiSampler::Nearest, QRhiSampler::Nearest, QRhiSampler::None,
341  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
342  sampler->create();
343 
344  // Store both
345  samplers.push_back({sampler, tex});
346  }
347 
349  r.state, vertexShader(), QString(rgb_filter).arg(filter));
350  }
351 
352  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
353  {
354  // Nothing particular, we just upload the whole buffer
355  setPixels(res, frame.data[0], frame.linesize[0]);
356  }
357 
358  void
359  setPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
360  {
361  const auto w = decoder.width, h = decoder.height;
362  auto y_tex = samplers[0].texture;
363 
364  QRhiTextureUploadEntry entry{
365  0, 0, createTextureUpload(pixels, w, h, bytes_per_pixel, stride)};
366 
367  QRhiTextureUploadDescription desc{entry};
368  res.uploadTexture(y_tex, desc);
369  }
370 };
371 }
Processes and renders a video frame on the GPU.
Definition: GPUVideoDecoder.hpp:43
static QRhiTextureSubresourceUploadDescription createTextureUpload(uint8_t *pixels, int w, int h, int bytesPerPixel, int stride)
Utility method to create a QRhiTextureSubresourceUploadDescription.
Definition: GPUVideoDecoder.cpp:22
List of nodes to be rendered to an output.
Definition: RenderList.hpp:19
RenderState & state
RenderState corresponding to this RenderList.
Definition: RenderList.hpp:89
Graphics rendering pipeline for ossia score.
Definition: Filter/PreviewWidget.hpp:12
std::pair< QShader, QShader > makeShaders(const RenderState &v, QString vert, QString frag)
Get a pair of compiled vertex / fragment shaders from GLSL 4.5 sources.
Definition: score-plugin-gfx/Gfx/Graph/Utils.cpp:342
Definition: VideoInterface.hpp:16
Definition: RGBA.hpp:10
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: RGBA.hpp:69
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: RGBA.hpp:45
Definition: RGBA.hpp:193
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: RGBA.hpp:268
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: RGBA.hpp:245
Definition: RGBA.hpp:90
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: RGBA.hpp:169
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: RGBA.hpp:128
Definition: RGBA.hpp:289
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: RGBA.hpp:352
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: RGBA.hpp:328