5 #include <libavcodec/avcodec.h>
6 #include <libavdevice/avdevice.h>
7 #include <libavformat/avformat.h>
8 #include <libavutil/pixdesc.h>
9 #include <libswresample/swresample.h>
10 #include <libswscale/swscale.h>
13 #include <Audio/Settings/Model.hpp>
14 #include <Gfx/Libav/AudioFrameEncoder.hpp>
15 #include <Gfx/Libav/LibavOutputSettings.hpp>
17 #include <score/application/ApplicationContext.hpp>
18 #include <score/tools/Debug.hpp>
20 #include <ossia/detail/flat_map.hpp>
22 #include <QApplication>
24 #include <CDSPResampler.h>
35 ossia::flat_map<std::string, std::string> options;
40 const AVCodec* codec{};
42 AVCodecContext* enc{};
48 AVFrame* cache_input_frame{};
53 struct SwsContext* sws_ctx{};
54 std::vector<std::unique_ptr<r8b::CDSPResampler>> resamplers;
56 std::unique_ptr<AudioFrameEncoder> encoder;
61 codec = avcodec_find_encoder_by_name(opts.codec.c_str());
64 qDebug() <<
"Could not find encoder for " << opts.codec.c_str();
68 this->tmp_pkt = av_packet_alloc();
71 qDebug() <<
"Could not allocate AVPacket";
75 this->st = avformat_new_stream(oc,
nullptr);
78 qDebug() <<
"Could not allocate stream";
81 this->st->id = oc->nb_streams - 1;
84 AVBufferRef* hw_ctx{};
88 AVHWDeviceType device = AV_HWDEVICE_TYPE_QSV;
89 int ret = av_hwdevice_ctx_create(&hw_ctx, device,
"auto",
nullptr, 0);
92 qDebug() <<
"Error while opening hardware encoder: " << av_to_string(ret);
97 this->enc = avcodec_alloc_context3(codec);
100 qDebug() <<
"Could not alloc an encoding context";
106 case AVMEDIA_TYPE_AUDIO:
107 init_audio(set, this->enc);
109 case AVMEDIA_TYPE_VIDEO:
110 init_video(set, this->enc);
118 if(oc->oformat->flags & AVFMT_GLOBALHEADER)
120 this->enc->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
126 #if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 24, 100)
127 c->sample_fmt = av_get_sample_fmt(set.audio_converted_smpfmt.toStdString().c_str());
129 if(codec->supported_samplerates)
131 c->sample_rate = codec->supported_samplerates[0];
132 for(
int i = 0; codec->supported_samplerates[i]; i++)
134 if(codec->supported_samplerates[i] == set.audio_sample_rate)
135 c->sample_rate = set.audio_sample_rate;
140 c->sample_rate = set.audio_sample_rate;
143 c->ch_layout.order = AV_CHANNEL_ORDER_UNSPEC;
144 c->ch_layout.nb_channels = set.audio_channels;
145 if(set.audio_encoder_short ==
"pcm_s24le" || set.audio_encoder_short ==
"pcm_s24be")
146 c->bits_per_raw_sample = 24;
148 this->st->time_base = AVRational{1, c->sample_rate};
149 c->time_base = AVRational{1, c->sample_rate};
150 c->framerate = AVRational{c->sample_rate, 1};
151 qDebug() <<
"Opening audio encoder with: rate: " << c->sample_rate;
157 c->codec_id = codec->id;
167 c->width = set.width;
168 c->height = set.height;
173 this->st->time_base = AVRational{100000, int(100000 * set.rate)};
174 c->time_base = this->st->time_base;
175 c->framerate = AVRational{this->st->time_base.den, this->st->time_base.num};
184 c->pix_fmt = av_get_pix_fmt(set.video_converted_pixfmt.toStdString().c_str());
185 c->strict_std_compliance = FF_COMPLIANCE_NORMAL;
186 if(c->codec_id == AV_CODEC_ID_MPEG2VIDEO)
191 if(c->codec_id == AV_CODEC_ID_MPEG1VIDEO)
202 AVDictionary* opt_arg)
204 #if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 24, 100)
205 AVDictionary* opt =
nullptr;
207 av_dict_copy(&opt, opt_arg, 0);
208 int ret = avcodec_open2(enc, codec, &opt);
212 qDebug() <<
"Could not open audio codec: " << av_to_string(ret);
217 if(enc->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE)
220 nb_samples = audio_stgs.getBufferSize();
221 enc->frame_size = nb_samples;
222 qDebug() <<
"Setting frame_size: " << nb_samples;
226 nb_samples = enc->frame_size;
227 qDebug() <<
"Forcing frame_size: " << nb_samples;
230 cache_input_frame = alloc_audio_frame(
231 enc->sample_fmt, &enc->ch_layout, enc->sample_rate, nb_samples);
234 ret = avcodec_parameters_from_context(this->st->codecpar, enc);
237 qDebug() <<
"Could not copy the stream parameters";
242 auto input_fmt = AV_SAMPLE_FMT_FLTP;
244 = av_get_sample_fmt(set.audio_converted_smpfmt.toStdString().c_str());
245 SCORE_ASSERT(input_fmt != -1);
246 SCORE_ASSERT(conv_fmt != -1);
250 const int input_sample_rate = ctx.getRate();
251 if(enc->sample_rate != input_sample_rate)
253 for(
int i = 0; i < set.audio_channels; i++)
254 this->resamplers.push_back(std::make_unique<r8b::CDSPResampler>(
255 input_sample_rate, enc->sample_rate, nb_samples * 2, 3.0, 206.91,
261 case AV_SAMPLE_FMT_NONE:
262 case AV_SAMPLE_FMT_U8:
263 case AV_SAMPLE_FMT_S16:
264 encoder = std::make_unique<S16IAudioFrameEncoder>(nb_samples);
266 case AV_SAMPLE_FMT_S32:
267 if(enc->bits_per_raw_sample == 24)
268 encoder = std::make_unique<S24IAudioFrameEncoder>(nb_samples);
270 encoder = std::make_unique<S32IAudioFrameEncoder>(nb_samples);
272 case AV_SAMPLE_FMT_FLT:
273 encoder = std::make_unique<FltIAudioFrameEncoder>(nb_samples);
275 case AV_SAMPLE_FMT_DBL:
276 encoder = std::make_unique<DblIAudioFrameEncoder>(nb_samples);
279 case AV_SAMPLE_FMT_U8P:
280 case AV_SAMPLE_FMT_S16P:
281 case AV_SAMPLE_FMT_S32P:
282 case AV_SAMPLE_FMT_FLTP:
283 encoder = std::make_unique<FltPAudioFrameEncoder>(nb_samples);
285 case AV_SAMPLE_FMT_DBLP:
286 case AV_SAMPLE_FMT_S64:
287 case AV_SAMPLE_FMT_S64P:
296 #if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 24, 100)
297 static AVFrame* alloc_audio_frame(
298 enum AVSampleFormat sample_fmt,
const AVChannelLayout* channel_layout,
299 int sample_rate,
int nb_samples)
301 AVFrame* frame = av_frame_alloc();
304 qDebug() <<
"Error allocating an audio frame";
308 frame->format = sample_fmt;
309 frame->ch_layout.order = channel_layout->order;
310 frame->ch_layout.nb_channels = channel_layout->nb_channels;
311 frame->sample_rate = sample_rate;
312 frame->nb_samples = nb_samples;
316 if(av_frame_get_buffer(frame, 0) < 0)
318 qDebug() <<
"Error allocating an audio buffer";
327 static AVFrame* alloc_video_frame(
enum AVPixelFormat pix_fmt,
int width,
int height)
329 auto frame = av_frame_alloc();
333 frame->format = pix_fmt;
334 frame->width = width;
335 frame->height = height;
338 const int ret = av_frame_get_buffer(frame, 0);
341 qDebug() <<
"Could not allocate frame data.";
350 AVDictionary* opt_arg)
352 AVCodecContext* c = this->enc;
353 AVDictionary* opt =
nullptr;
355 av_dict_copy(&opt, opt_arg, 0);
358 int err = av_opt_set_double(this->enc->priv_data,
"crf", 0.0, 0);
361 qDebug() <<
"failed to initialize encoder: " << av_to_string(err);
365 SCORE_ASSERT(this->enc->flags & AV_CODEC_FLAG_GLOBAL_HEADER);
366 int ret = avcodec_open2(this->enc, codec, &opt);
370 qDebug() <<
"Could not open video codec: " << av_to_string(ret);
375 this->cache_input_frame = alloc_video_frame(AV_PIX_FMT_RGBA, c->width, c->height);
376 if(!this->cache_input_frame)
378 qDebug() <<
"Could not allocate video frame";
382 this->tmp_frame =
nullptr;
386 auto input_fmt = av_get_pix_fmt(set.video_render_pixfmt.toStdString().c_str());
387 auto conv_fmt = av_get_pix_fmt(set.video_converted_pixfmt.toStdString().c_str());
388 SCORE_ASSERT(input_fmt != -1);
389 SCORE_ASSERT(conv_fmt != -1);
390 sws_ctx = sws_getContext(
391 set.width, set.height, input_fmt, set.width, set.height, conv_fmt, 1,
nullptr,
393 SCORE_ASSERT(sws_ctx);
394 this->tmp_frame = alloc_video_frame(conv_fmt, c->width, c->height);
397 qDebug() <<
"Could not allocate temporary video frame";
403 ret = avcodec_parameters_from_context(this->st->codecpar, c);
406 qDebug() <<
"Could not copy the stream parameters";
413 AVDictionary* opt_arg)
417 SCORE_ASSERT(opt_arg);
418 if(codec->type == AVMEDIA_TYPE_AUDIO)
420 open_audio(set, oc, codec, opt_arg);
422 else if(codec->type == AVMEDIA_TYPE_VIDEO)
424 open_video(set, oc, codec, opt_arg);
428 void close(AVFormatContext* oc)
430 avcodec_free_context(&enc);
431 av_frame_free(&cache_input_frame);
432 av_frame_free(&tmp_frame);
433 av_packet_free(&tmp_pkt);
434 sws_freeContext(sws_ctx);
438 AVFrame* get_video_frame()
442 if(av_frame_make_writable(this->cache_input_frame) < 0)
445 this->cache_input_frame->pts = this->next_pts++;
447 return this->cache_input_frame;
450 AVFrame* get_audio_frame()
454 if(av_frame_make_writable(this->cache_input_frame) < 0)
457 this->cache_input_frame->pts = this->next_pts;
458 this->next_pts += this->enc->frame_size;
460 return this->cache_input_frame;
463 int write_video_frame(AVFormatContext* fmt_ctx, AVFrame* input_frame)
465 #if LIBSWSCALE_VERSION_INT >= AV_VERSION_INT(7, 5, 100)
467 int ret = sws_scale_frame(sws_ctx, tmp_frame, input_frame);
470 qDebug() <<
"Error during sws_scale_frame: " << av_to_string(ret);
474 tmp_frame->quality = FF_LAMBDA_MAX;
475 tmp_frame->pict_type = AV_PICTURE_TYPE_I;
479 ret = avcodec_send_frame(enc, tmp_frame);
482 qDebug() <<
"Error sending a frame to the encoder: " << av_to_string(ret);
488 ret = avcodec_receive_packet(enc, tmp_pkt);
489 if(ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
493 qDebug() <<
"Error encoding a frame: " << av_to_string(ret);
498 av_packet_rescale_ts(tmp_pkt, enc->time_base, st->time_base);
499 tmp_pkt->stream_index = st->index;
500 tmp_pkt->flags |= AV_PKT_FLAG_KEY;
502 ret = av_interleaved_write_frame(fmt_ctx, tmp_pkt);
505 qDebug() <<
"Error while writing output packet: " << av_to_string(ret);
510 return ret == AVERROR_EOF ? 1 : 0;
532 int write_audio_frame(AVFormatContext* fmt_ctx, AVFrame* input_frame)
535 int ret = avcodec_send_frame(enc, input_frame);
538 qDebug() <<
"Error sending a frame to the encoder: " << av_to_string(ret);
544 ret = avcodec_receive_packet(enc, tmp_pkt);
545 if(ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
549 qDebug() <<
"Error encoding a frame: " << av_to_string(ret);
554 av_packet_rescale_ts(tmp_pkt, enc->time_base, st->time_base);
555 tmp_pkt->stream_index = st->index;
557 ret = av_interleaved_write_frame(fmt_ctx, tmp_pkt);
560 qDebug() <<
"Error while writing output packet: " << av_to_string(ret);
565 return ret == AVERROR_EOF ? 1 : 0;
Definition: score-plugin-audio/Audio/Settings/Model.hpp:22
Binds the rendering pipeline to ossia processes.
Definition: CameraDevice.cpp:28
Definition: LibavOutputSettings.hpp:16
Definition: LibavOutputStream.hpp:39
Definition: LibavOutputStream.hpp:32
T & settings() const
Access a specific Settings model instance.
Definition: ApplicationContext.hpp:40