RGBA.hpp
1 #pragma once
2 #include <Gfx/Graph/decoders/GPUVideoDecoder.hpp>
3 extern "C" {
4 #include <libavformat/avformat.h>
5 }
6 
7 namespace score::gfx
8 {
10 {
11  static const constexpr auto rgb_filter = R"_(#version 450
12 
13 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
14 
15  layout(binding=3) uniform sampler2D y_tex;
16 
17  layout(location = 0) in vec2 v_texcoord;
18  layout(location = 0) out vec4 fragColor;
19 
20  vec4 processTexture(vec4 tex) {
21  vec4 processed = tex;
22  { %1 }
23  return processed;
24  }
25 
26  void main ()
27  {
28  fragColor = processTexture(texture(y_tex, v_texcoord));
29  })_";
30 
32  QRhiTexture::Format fmt, int bytes_per_pixel, Video::ImageFormat& d,
33  QString f = "")
34  : format{fmt}
35  , bytes_per_pixel{bytes_per_pixel}
36  , decoder{d}
37  , filter{std::move(f)}
38  {
39  }
40  QRhiTexture::Format format;
41  int bytes_per_pixel{}; // bpp/8 !
42  Video::ImageFormat& decoder;
43  QString filter;
44 
45  std::pair<QShader, QShader> init(RenderList& r) override
46  {
47  auto& rhi = *r.state.rhi;
48  const auto w = decoder.width, h = decoder.height;
49 
50  {
51  // Create a texture
52  auto tex = rhi.newTexture(format, QSize{w, h}, 1, QRhiTexture::Flag{});
53  tex->create();
54 
55  // Create a sampler
56  auto sampler = rhi.newSampler(
57  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
58  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
59  sampler->create();
60 
61  // Store both
62  samplers.push_back({sampler, tex});
63  }
64 
66  r.state, vertexShader(), QString(rgb_filter).arg(filter));
67  }
68 
69  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
70  {
71  // Nothing particular, we just upload the whole buffer
72  setPixels(res, frame.data[0], frame.linesize[0]);
73  }
74 
75  void
76  setPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
77  {
78  const auto w = decoder.width, h = decoder.height;
79  auto y_tex = samplers[0].texture;
80 
81  QRhiTextureUploadEntry entry{
82  0, 0, createTextureUpload(pixels, w, h, bytes_per_pixel, stride)};
83 
84  QRhiTextureUploadDescription desc{entry};
85  res.uploadTexture(y_tex, desc);
86  }
87 };
88 
90 {
91  static const constexpr auto rgb_filter = R"_(#version 450
92 
93 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
94 
95  %1
96  layout(location = 0) in vec2 v_texcoord;
97  layout(location = 0) out vec4 fragColor;
98 
99  vec4 processTexture(vec4 tex) {
100  vec4 processed = tex;
101  { %2 }
102  return processed;
103  }
104 
105  void main ()
106  {
107  vec4 tex = vec4(1.);
108  %3
109  fragColor = processTexture(tex);
110  })_";
111 
113  QRhiTexture::Format fmt, int bytes_per_pixel, QString planes,
114  Video::ImageFormat& d, QString f = "")
115  : format{fmt}
116  , bytes_per_pixel{bytes_per_pixel}
117  , planes{planes}
118  , decoder{d}
119  , filter{std::move(f)}
120  {
121  }
122  QRhiTexture::Format format;
123  int bytes_per_pixel{}; // bpp/8 !
124  QString planes{};
125  Video::ImageFormat& decoder;
126  QString filter;
127 
128  std::pair<QShader, QShader> init(RenderList& r) override
129  {
130  auto& rhi = *r.state.rhi;
131  const auto w = decoder.width, h = decoder.height;
132 
133  QString samplers_code;
134  QString read_texture_code;
135 
136  const int binding_orig = 3;
137  for(int i = 0; i < planes.size(); i++)
138  {
139  samplers_code += QString(" layout(binding=%1) uniform sampler2D t%2;\n")
140  .arg(binding_orig + i)
141  .arg(i);
142  read_texture_code += QString(" tex.%1 = texture(t%2, v_texcoord).r;\n")
143  .arg(planes[i])
144  .arg(i);
145 
146  // Create a texture
147  auto tex = rhi.newTexture(format, QSize{w, h}, 1, QRhiTexture::Flag{});
148  tex->create();
149 
150  // Create a sampler
151  auto sampler = rhi.newSampler(
152  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
153  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
154  sampler->create();
155 
156  // Store both
157  samplers.push_back({sampler, tex});
158  }
159 
161  r.state, vertexShader(),
162  QString(rgb_filter).arg(samplers_code).arg(filter).arg(read_texture_code));
163  }
164 
165  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
166  {
167  // Nothing particular, we just upload the whole buffer
168  for(int i = 0; i < planes.size(); i++)
169  {
170  setPixels(res, samplers[i].texture, frame.data[i], frame.linesize[i]);
171  }
172  }
173 
174  void setPixels(
175  QRhiResourceUpdateBatch& res, QRhiTexture* tex, uint8_t* pixels,
176  int stride) const noexcept
177  {
178  const auto w = decoder.width, h = decoder.height;
179 
180  QRhiTextureUploadEntry entry{
181  0, 0, createTextureUpload(pixels, w, h, bytes_per_pixel, stride)};
182 
183  QRhiTextureUploadDescription desc{entry};
184  res.uploadTexture(tex, desc);
185  }
186 };
187 
189 {
190  static const constexpr auto rgb_filter = R"_(#version 450
191 
192 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
193 
194  layout(binding=3) uniform sampler2DRect y_tex;
195 
196  layout(location = 0) in vec2 v_texcoord;
197  layout(location = 0) out vec4 fragColor;
198 
199  vec4 processTexture(vec4 tex) {
200  vec4 processed = tex;
201  { %1 }
202  return processed;
203  }
204 
205  void main ()
206  {
207  fragColor = processTexture(texture(y_tex, v_texcoord));
208  })_";
209 
210  static constexpr const char* vertex = R"_(#version 450
211 layout(location = 0) in vec2 position;
212 layout(location = 1) in vec2 texcoord;
213 
214 layout(location = 0) out vec2 v_texcoord;
215 
216 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
217 
218 out gl_PerVertex { vec4 gl_Position; };
219 
220 void main()
221 {
222  v_texcoord = texcoord * mat.texSz.xy;
223  gl_Position = renderer.clipSpaceCorrMatrix * vec4(position.x * mat.scale.x, position.y * mat.scale.y, 0.0, 1.);
224 }
225 )_";
226 
228  QRhiTexture::Format fmt, int bytes_per_pixel, Video::ImageFormat& d,
229  QString f = "")
230  : format{fmt}
231  , bytes_per_pixel{bytes_per_pixel}
232  , decoder{d}
233  , filter{std::move(f)}
234  {
235  }
236  QRhiTexture::Format format;
237  int bytes_per_pixel{}; // bpp/8 !
238  Video::ImageFormat& decoder;
239  QString filter;
240 
241  std::pair<QShader, QShader> init(RenderList& r) override
242  {
243  auto& rhi = *r.state.rhi;
244  const auto w = decoder.width, h = decoder.height;
245 
246  {
247  // Create a texture
248  auto tex = rhi.newTexture(format, QSize{w, h}, 1, QRhiTexture::Flag{});
249  tex->create();
250 
251  // Create a sampler
252  auto sampler = rhi.newSampler(
253  QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
254  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
255  sampler->create();
256 
257  // Store both
258  samplers.push_back({sampler, tex});
259  }
260 
261  return score::gfx::makeShaders(r.state, vertex, QString(rgb_filter).arg(filter));
262  }
263 
264  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
265  {
266  // Nothing particular, we just upload the whole buffer
267  setPixels(res, frame.data[0], frame.linesize[0]);
268  }
269 
270  void
271  setPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
272  {
273  const auto w = decoder.width, h = decoder.height;
274  auto y_tex = samplers[0].texture;
275 
276  QRhiTextureUploadEntry entry{
277  0, 0, createTextureUpload(pixels, w, h, bytes_per_pixel, stride)};
278 
279  QRhiTextureUploadDescription desc{entry};
280  res.uploadTexture(y_tex, desc);
281  }
282 };
283 
285 {
286  static const constexpr auto rgb_filter = R"_(#version 450
287 
288 )_" SCORE_GFX_VIDEO_UNIFORMS R"_(
289 
290  layout(binding=3) uniform sampler2D y_tex;
291 
292  layout(location = 0) in vec2 v_texcoord;
293  layout(location = 0) out vec4 fragColor;
294 
295  vec4 processTexture(vec4 tex) {
296  vec4 processed = tex;
297  { %1 }
298  return processed;
299  }
300 
301  void main ()
302  {
303  float w = mat.texSz.x;
304  float h = mat.texSz.y;
305  int x = int(floor(v_texcoord.x * w) * 3.);
306  int y = int(v_texcoord.y * h);
307  float r = texelFetch(y_tex, ivec2(x + 0, y), 0).r;
308  float g = texelFetch(y_tex, ivec2(x + 1, y), 0).r;
309  float b = texelFetch(y_tex, ivec2(x + 2, y), 0).r;
310  fragColor = processTexture(vec4(r, g, b, 1.));
311  })_";
312 
313  RGB24Decoder(Video::ImageFormat& d, QString f = "")
314  : bytes_per_pixel{3}
315  , decoder{d}
316  , filter{std::move(f)}
317  {
318  }
319  QRhiTexture::Format format;
320  int bytes_per_pixel{}; // bpp/8 !
321  Video::ImageFormat& decoder;
322  QString filter;
323 
324  std::pair<QShader, QShader> init(RenderList& r) override
325  {
326  auto& rhi = *r.state.rhi;
327  const auto w = decoder.width, h = decoder.height;
328 
329  {
330  // Create a texture
331  auto tex = rhi.newTexture(QRhiTexture::R8, QSize{w * 3, h}, 1, QRhiTexture::sRGB);
332  tex->create();
333 
334  // Create a sampler
335  auto sampler = rhi.newSampler(
336  QRhiSampler::Nearest, QRhiSampler::Nearest, QRhiSampler::None,
337  QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge);
338  sampler->create();
339 
340  // Store both
341  samplers.push_back({sampler, tex});
342  }
343 
345  r.state, vertexShader(), QString(rgb_filter).arg(filter));
346  }
347 
348  void exec(RenderList&, QRhiResourceUpdateBatch& res, AVFrame& frame) override
349  {
350  // Nothing particular, we just upload the whole buffer
351  setPixels(res, frame.data[0], frame.linesize[0]);
352  }
353 
354  void
355  setPixels(QRhiResourceUpdateBatch& res, uint8_t* pixels, int stride) const noexcept
356  {
357  const auto w = decoder.width, h = decoder.height;
358  auto y_tex = samplers[0].texture;
359 
360  QRhiTextureUploadEntry entry{
361  0, 0, createTextureUpload(pixels, w, h, bytes_per_pixel, stride)};
362 
363  QRhiTextureUploadDescription desc{entry};
364  res.uploadTexture(y_tex, desc);
365  }
366 };
367 }
Processes and renders a video frame on the GPU.
Definition: GPUVideoDecoder.hpp:43
static QRhiTextureSubresourceUploadDescription createTextureUpload(uint8_t *pixels, int w, int h, int bytesPerPixel, int stride)
Utility method to create a QRhiTextureSubresourceUploadDescription.
Definition: GPUVideoDecoder.cpp:22
List of nodes to be rendered to an output.
Definition: RenderList.hpp:19
RenderState & state
RenderState corresponding to this RenderList.
Definition: RenderList.hpp:89
Graphics rendering pipeline for ossia score.
Definition: Filter/PreviewWidget.hpp:12
std::pair< QShader, QShader > makeShaders(const RenderState &v, QString vert, QString frag)
Get a pair of compiled vertex / fragment shaders from GLSL 4.5 sources.
Definition: score-plugin-gfx/Gfx/Graph/Utils.cpp:342
Definition: VideoInterface.hpp:16
Definition: RGBA.hpp:10
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: RGBA.hpp:69
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: RGBA.hpp:45
Definition: RGBA.hpp:189
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: RGBA.hpp:264
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: RGBA.hpp:241
Definition: RGBA.hpp:90
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: RGBA.hpp:165
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: RGBA.hpp:128
Definition: RGBA.hpp:285
void exec(RenderList &, QRhiResourceUpdateBatch &res, AVFrame &frame) override
Decode and upload a video frame to the GPU.
Definition: RGBA.hpp:348
std::pair< QShader, QShader > init(RenderList &r) override
Initialize a GPUVideoDecoder.
Definition: RGBA.hpp:324