agsantos | 4bb4bc5 | 2021-03-08 14:21:45 -0500 | [diff] [blame^] | 1 | /** |
| 2 | * Copyright (C) 2021 Savoir-faire Linux Inc. |
| 3 | * |
| 4 | * Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License as published by |
| 8 | * the Free Software Foundation; either version 3 of the License, or |
| 9 | * (at your option) any later version. |
| 10 | * |
| 11 | * This program is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | * GNU General Public License for more details. |
| 15 | * |
| 16 | * You should have received a copy of the GNU General Public License |
| 17 | * along with this program; if not, write to the Free Software |
| 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 19 | */ |
| 20 | |
| 21 | #include "WatermarkVideoSubscriber.h" |
| 22 | |
| 23 | extern "C" { |
| 24 | #include <libavutil/display.h> |
| 25 | } |
| 26 | #include <accel.h> |
| 27 | #include <frameScaler.h> |
| 28 | |
| 29 | #include <pluglog.h> |
| 30 | #include <algorithm> |
| 31 | #include <ctime> |
| 32 | #include <clocale> |
| 33 | #include <iostream> |
| 34 | |
| 35 | const std::string TAG = "Watermark"; |
| 36 | const char sep = separator(); |
| 37 | |
| 38 | namespace jami { |
| 39 | |
| 40 | WatermarkVideoSubscriber::WatermarkVideoSubscriber(const std::string& dataPath, |
| 41 | std::map<std::string, std::string>& preferences) |
| 42 | { |
| 43 | if (std::setlocale(LC_TIME, std::locale("").name().c_str()) == NULL) { |
| 44 | Plog::log(Plog::LogPriority::INFO, TAG, "error while setting locale"); |
| 45 | } |
| 46 | fontFile_ = dataPath + sep + "Muli-Light.ttf"; |
| 47 | #ifdef WIN32 |
| 48 | for (int i = fontFile_.size(); i > 0; i--) |
| 49 | if (fontFile_[i] == '\\') |
| 50 | fontFile_.insert(i, "\\"); |
| 51 | fontFile_.insert(1, "\\"); |
| 52 | #endif |
| 53 | try { |
| 54 | setParameter(preferences["fontsize"], Parameter::FONTSIZE); |
| 55 | setParameter(preferences["logosize"], Parameter::LOGOSIZE); |
| 56 | setParameter(preferences["markbackground"], Parameter::LOGOBACKGROUND); |
| 57 | setParameter(preferences["showinfos"], Parameter::SHOWINFOS); |
| 58 | setParameter(preferences["showlogo"], Parameter::SHOWLOGO); |
| 59 | setParameter(preferences["mark"], Parameter::LOGOPATH); |
| 60 | setParameter(preferences["date"], Parameter::DATE); |
| 61 | setParameter(preferences["dateformat"], Parameter::DATEFORMAT); |
| 62 | setParameter(preferences["time"], Parameter::TIME); |
| 63 | setParameter(preferences["timezone"], Parameter::TIMEZONE); |
| 64 | setParameter(preferences["timeformat"], Parameter::TIMEFORMAT); |
| 65 | setParameter(preferences["location"], Parameter::LOCATION); |
| 66 | setParameter(preferences["infosposition"], Parameter::INFOSPOSITION); |
| 67 | setParameter(preferences["logoposition"], Parameter::LOGOPOSITION); |
| 68 | } catch (std::exception e) { |
| 69 | Plog::log(Plog::LogPriority::ERR, TAG, e.what()); |
| 70 | } |
| 71 | } |
| 72 | |
| 73 | WatermarkVideoSubscriber::~WatermarkVideoSubscriber() |
| 74 | { |
| 75 | validLogo_ = false; |
| 76 | logoFilter_.clean(); |
| 77 | detach(); |
| 78 | std::lock_guard<std::mutex> lk(mtx_); |
| 79 | Plog::log(Plog::LogPriority::INFO, TAG, "~WatermarkMediaProcessor"); |
| 80 | } |
| 81 | |
| 82 | MediaStream |
| 83 | WatermarkVideoSubscriber::getLogoAVFrameInfos() |
| 84 | { |
| 85 | AVFormatContext* ctx = avformat_alloc_context(); |
| 86 | |
| 87 | // Open |
| 88 | if (avformat_open_input(&ctx, logoPath_.c_str(), NULL, NULL) != 0) { |
| 89 | avformat_free_context(ctx); |
| 90 | Plog::log(Plog::LogPriority::INFO, TAG, "Couldn't open input stream."); |
| 91 | return {}; |
| 92 | } |
| 93 | pFormatCtx_.reset(ctx); |
| 94 | // Retrieve stream information |
| 95 | if (avformat_find_stream_info(pFormatCtx_.get(), NULL) < 0) { |
| 96 | Plog::log(Plog::LogPriority::INFO, TAG, "Couldn't find stream information."); |
| 97 | return {}; |
| 98 | } |
| 99 | |
| 100 | // Dump valid information onto standard error |
| 101 | av_dump_format(pFormatCtx_.get(), 0, logoPath_.c_str(), false); |
| 102 | |
| 103 | // Find the video stream |
| 104 | for (int i = 0; i < static_cast<int>(pFormatCtx_->nb_streams); i++) |
| 105 | if (pFormatCtx_->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { |
| 106 | videoStream_ = i; |
| 107 | break; |
| 108 | } |
| 109 | |
| 110 | if (videoStream_ == -1) { |
| 111 | Plog::log(Plog::LogPriority::INFO, TAG, "Didn't find a video stream."); |
| 112 | return {}; |
| 113 | } |
| 114 | |
| 115 | rational<int> fr = pFormatCtx_->streams[videoStream_]->r_frame_rate; |
| 116 | return MediaStream("logo", |
| 117 | pFormatCtx_->streams[videoStream_]->codecpar->format, |
| 118 | 1 / fr, |
| 119 | pFormatCtx_->streams[videoStream_]->codecpar->width, |
| 120 | pFormatCtx_->streams[videoStream_]->codecpar->height, |
| 121 | 0, |
| 122 | fr); |
| 123 | } |
| 124 | |
| 125 | void |
| 126 | WatermarkVideoSubscriber::loadMarkLogo() |
| 127 | { |
| 128 | if (logoPath_.empty()) |
| 129 | return; |
| 130 | |
| 131 | logoFilter_.clean(); |
| 132 | logoDescription_ = "[logo]scale=" + logoSize_ + "*" + std::to_string(pluginFrameSize_.first) |
| 133 | + ":" + logoSize_ + "*" + std::to_string(pluginFrameSize_.second) |
| 134 | + ":force_original_aspect_ratio='decrease',format=yuva444p," |
| 135 | "split=2[bg][fg],[bg]drawbox=c='" |
| 136 | + backgroundColor_ |
| 137 | + "':replace=1:t=fill[bg]," |
| 138 | "[bg][fg]overlay=format=auto"; |
| 139 | Plog::log(Plog::LogPriority::INFO, TAG, logoDescription_); |
| 140 | logoStream_ = getLogoAVFrameInfos(); |
| 141 | logoFilter_.initialize(logoDescription_, {logoStream_}); |
| 142 | |
| 143 | int got_frame; |
| 144 | AVCodecContext* pCodecCtx; |
| 145 | AVCodec* pCodec; |
| 146 | AVPacket* packet; |
| 147 | |
| 148 | pCodecCtx = pFormatCtx_->streams[videoStream_]->codec; |
| 149 | pCodec = avcodec_find_decoder(pCodecCtx->codec_id); |
| 150 | if (pCodec == nullptr) { |
| 151 | pFormatCtx_.reset(); |
| 152 | Plog::log(Plog::LogPriority::INFO, TAG, "Codec not found."); |
| 153 | validLogo_ = false; |
| 154 | return; |
| 155 | } |
| 156 | |
| 157 | // Open codec |
| 158 | if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0) { |
| 159 | pFormatCtx_.reset(); |
| 160 | Plog::log(Plog::LogPriority::INFO, TAG, "Could not open codec."); |
| 161 | validLogo_ = false; |
| 162 | return; |
| 163 | } |
| 164 | |
| 165 | packet = av_packet_alloc(); |
| 166 | if (av_read_frame(pFormatCtx_.get(), packet) < 0) { |
| 167 | avcodec_close(pCodecCtx); |
| 168 | av_packet_free(&packet); |
| 169 | pFormatCtx_.reset(); |
| 170 | Plog::log(Plog::LogPriority::INFO, TAG, "Could not read packet from context."); |
| 171 | validLogo_ = false; |
| 172 | return; |
| 173 | } |
| 174 | |
| 175 | AVFrame* logoImage = av_frame_alloc(); |
| 176 | avcodec_decode_video2(pCodecCtx, logoImage, &got_frame, packet); |
| 177 | if (got_frame) { |
| 178 | logoFilter_.feedInput(logoImage, "logo"); |
| 179 | } |
| 180 | logoFilter_.feedEOF("logo"); |
| 181 | |
| 182 | frameFree(logoImage); |
| 183 | avcodec_close(pCodecCtx); |
| 184 | av_packet_free(&packet); |
| 185 | pFormatCtx_.reset(); |
| 186 | mark_.reset(logoFilter_.readOutput()); |
| 187 | mark_->pts = 0; |
| 188 | mark_->best_effort_timestamp = 0; |
| 189 | validLogo_ = mark_->width && mark_->height; |
| 190 | } |
| 191 | |
| 192 | void |
| 193 | WatermarkVideoSubscriber::setParameter(std::string& parameter, Parameter type) |
| 194 | { |
| 195 | switch (type) { |
| 196 | case (Parameter::LOGOSIZE): |
| 197 | logoSize_ = parameter; |
| 198 | break; |
| 199 | case (Parameter::TIMEZONE): |
| 200 | timeZone_ = parameter == "1"; |
| 201 | return; |
| 202 | case (Parameter::FONTSIZE): |
| 203 | fontSize_ = std::stoi(parameter); |
| 204 | break; |
| 205 | case (Parameter::LOGOBACKGROUND): |
| 206 | backgroundColor_ = parameter; |
| 207 | if (backgroundColor_.find("black") == std::string::npos) { |
| 208 | fontColor_ = "black"; |
| 209 | fontBackground_ = "white@0.5"; |
| 210 | } else { |
| 211 | fontColor_ = "white"; |
| 212 | fontBackground_ = "black@0.5"; |
| 213 | } |
| 214 | if (!logoPath_.empty()) |
| 215 | setParameter(logoPath_, Parameter::LOGOPATH); |
| 216 | break; |
| 217 | case (Parameter::SHOWLOGO): |
| 218 | showLogo_ = parameter == "1"; |
| 219 | break; |
| 220 | case (Parameter::SHOWINFOS): |
| 221 | showInfos_ = parameter == "1"; |
| 222 | break; |
| 223 | case (Parameter::LOGOPATH): |
| 224 | logoPath_ = parameter; |
| 225 | break; |
| 226 | case (Parameter::TIME): |
| 227 | time_ = parameter == "1"; |
| 228 | break; |
| 229 | case (Parameter::DATE): |
| 230 | date_ = parameter == "1"; |
| 231 | break; |
| 232 | case (Parameter::TIMEFORMAT): |
| 233 | timeFormat_ = "%{localtime\\:'" + parameter + "'}"; |
| 234 | if (timeZone_) |
| 235 | timeFormat_ = "%{localtime\\:'" + parameter + " %Z'}"; |
| 236 | break; |
| 237 | case (Parameter::DATEFORMAT): |
| 238 | dateFormat_ = "%{localtime\\:'" + parameter + "'}"; |
| 239 | break; |
| 240 | case (Parameter::LOCATION): |
| 241 | location_ = parameter; |
| 242 | break; |
| 243 | case (Parameter::INFOSPOSITION): |
| 244 | infosposition_ = parameter; |
| 245 | break; |
| 246 | case (Parameter::LOGOPOSITION): |
| 247 | logoposition_ = parameter; |
| 248 | break; |
| 249 | default: |
| 250 | return; |
| 251 | } |
| 252 | |
| 253 | firstRun = true; |
| 254 | } |
| 255 | |
| 256 | void |
| 257 | WatermarkVideoSubscriber::setFilterDescription() |
| 258 | { |
| 259 | loadMarkLogo(); |
| 260 | |
| 261 | std::string infoSep = ", "; |
| 262 | if (pluginFrameSize_.first < pluginFrameSize_.second) |
| 263 | infoSep = ",\n"; |
| 264 | |
| 265 | std::vector<std::string> infos; |
| 266 | infosSize_ = 0; |
| 267 | if (!location_.empty()) { |
| 268 | infosSize_++; |
| 269 | infos.emplace_back(location_); |
| 270 | } |
| 271 | if (date_) { |
| 272 | infosSize_++; |
| 273 | infos.emplace_back(dateFormat_); |
| 274 | } |
| 275 | if (time_) { |
| 276 | infosSize_++; |
| 277 | infos.emplace_back(timeFormat_); |
| 278 | } |
| 279 | infosString.clear(); |
| 280 | for (int i = 0; i < infosSize_ - 1; i++) |
| 281 | infosString += infos[i] + infoSep; |
| 282 | if (infosSize_ > 0) |
| 283 | infosString += infos.back(); |
| 284 | |
| 285 | setMarkPosition(); |
| 286 | |
| 287 | std::string rotateSides = ""; |
| 288 | if (std::abs(angle_) == 90) |
| 289 | rotateSides = ":out_w=ih:out_h=iw"; |
| 290 | |
| 291 | if (angle_ != 0) |
| 292 | pluginFilterDescription_ = "[input]rotate=" + rotation[angle_] + rotateSides |
| 293 | + "[rot],[rot][mark]overlay=" + std::to_string(points_[0].first) |
| 294 | + ":" + std::to_string(points_[0].second) |
| 295 | + ",rotate=" + rotation[-angle_] + rotateSides; |
| 296 | else |
| 297 | pluginFilterDescription_ = "[input][mark]overlay=" + std::to_string(points_[0].first) + ":" |
| 298 | + std::to_string(points_[0].second); |
| 299 | |
| 300 | std::string baseInfosDescription = "[input]rotate=" + rotation[angle_] + rotateSides |
| 301 | + ",drawtext=fontfile='" + fontFile_ + "':text='" |
| 302 | + infosString + "':fontcolor=" + fontColor_ |
| 303 | + ":fontsize=" + std::to_string(fontSize_) |
| 304 | + ":line_spacing=" + std::to_string(lineSpacing_) |
| 305 | + ":box=1:boxcolor=" + fontBackground_ + ":boxborderw=5:x="; |
| 306 | |
| 307 | if (infosposition_ == "1") |
| 308 | infosDescription_ = baseInfosDescription + std::to_string(points_[1].first) |
| 309 | + "-text_w:y=" + std::to_string(points_[1].second); |
| 310 | else if (infosposition_ == "2") |
| 311 | infosDescription_ = baseInfosDescription + std::to_string(points_[1].first) |
| 312 | + ":y=" + std::to_string(points_[1].second); |
| 313 | else if (infosposition_ == "3") |
| 314 | infosDescription_ = baseInfosDescription + std::to_string(points_[1].first) |
| 315 | + ":y=" + std::to_string(points_[1].second) + "-text_h"; |
| 316 | else if (infosposition_ == "4") |
| 317 | infosDescription_ = baseInfosDescription + std::to_string(points_[1].first) |
| 318 | + "-text_w:y=" + std::to_string(points_[1].second) + "-text_h"; |
| 319 | infosDescription_ += ",rotate=" + rotation[-angle_] + rotateSides + ",format=rgb24"; |
| 320 | |
| 321 | Plog::log(Plog::LogPriority::INFO, TAG, infosDescription_); |
| 322 | Plog::log(Plog::LogPriority::INFO, TAG, pluginFilterDescription_); |
| 323 | } |
| 324 | |
| 325 | void |
| 326 | WatermarkVideoSubscriber::setMarkPosition() |
| 327 | { |
| 328 | // 1, 2, 3, and 4 are cartesian positions |
| 329 | int margin = 10; |
| 330 | int markWidth = showLogo_ ? mark_->width : 0; |
| 331 | int markHeight = showLogo_ ? mark_->height : 0; |
| 332 | int infoHeight = (std::abs(angle_) == 90) ? (fontSize_ + lineSpacing_) * infosSize_ |
| 333 | : lineSpacing_ * 2 + fontSize_; |
| 334 | if (pluginFrameSize_.first == 0 || pluginFrameSize_.second == 0) |
| 335 | return; |
| 336 | |
| 337 | if (infosposition_ == "1") { |
| 338 | points_[1] = {pluginFrameSize_.first - margin, margin}; |
| 339 | } else if (infosposition_ == "2") { |
| 340 | points_[1] = {margin, margin}; |
| 341 | } else if (infosposition_ == "3") { |
| 342 | points_[1] = {margin, pluginFrameSize_.second - margin}; |
| 343 | } else if (infosposition_ == "4") { |
| 344 | points_[1] = {pluginFrameSize_.first - margin, pluginFrameSize_.second - margin}; |
| 345 | } |
| 346 | if (logoposition_ == "1") { |
| 347 | points_[0] = {pluginFrameSize_.first - mark_->width - margin / 2, margin}; |
| 348 | } else if (logoposition_ == "2") { |
| 349 | points_[0] = {margin / 2, margin}; |
| 350 | } else if (logoposition_ == "3") { |
| 351 | points_[0] = {margin / 2, pluginFrameSize_.second - markHeight - margin}; |
| 352 | } else if (logoposition_ == "4") { |
| 353 | points_[0] = {pluginFrameSize_.first - markWidth - margin / 2, |
| 354 | pluginFrameSize_.second - markHeight - margin}; |
| 355 | } |
| 356 | |
| 357 | if (infosposition_ == logoposition_ && showInfos_ && showLogo_) { |
| 358 | if (logoposition_ == "1" || logoposition_ == "2") { |
| 359 | points_[0].second += infoHeight; |
| 360 | } else if (logoposition_ == "3" || logoposition_ == "4") { |
| 361 | points_[0].second -= infoHeight; |
| 362 | } |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | void |
| 367 | WatermarkVideoSubscriber::update(jami::Observable<AVFrame*>*, AVFrame* const& pluginFrame) |
| 368 | { |
| 369 | if (!observable_ || !pluginFrame || (!validLogo_ && showLogo_)) |
| 370 | return; |
| 371 | |
| 372 | AVFrameSideData* side_data = av_frame_get_side_data(pluginFrame, AV_FRAME_DATA_DISPLAYMATRIX); |
| 373 | int newAngle {0}; |
| 374 | if (side_data) { |
| 375 | auto matrix_rotation = reinterpret_cast<int32_t*>(side_data->data); |
| 376 | newAngle = static_cast<int>(av_display_rotation_get(matrix_rotation)); |
| 377 | } |
| 378 | if (newAngle != angle_) { |
| 379 | angle_ = newAngle; |
| 380 | firstRun = true; |
| 381 | } |
| 382 | |
| 383 | //====================================================================================== |
| 384 | // GET RAW FRAME |
| 385 | wmFrame rgbFrame = {transferToMainMemory(pluginFrame, AV_PIX_FMT_NV12), frameFree}; |
| 386 | rgbFrame.reset(FrameScaler::convertFormat(rgbFrame.get(), AV_PIX_FMT_RGB24)); |
| 387 | if (!rgbFrame.get()) |
| 388 | return; |
| 389 | rgbFrame->pts = 1; |
| 390 | |
| 391 | if (firstRun) { |
| 392 | pluginFilter_.clean(); |
| 393 | infosFilter_.clean(); |
| 394 | pluginFrameSize_ = {rgbFrame->width, rgbFrame->height}; |
| 395 | if (std::abs(angle_) == 90) |
| 396 | pluginFrameSize_ = {rgbFrame->height, rgbFrame->width}; |
| 397 | |
| 398 | setFilterDescription(); |
| 399 | |
| 400 | rational<int> fr(rgbFrame->pts, 1); |
| 401 | pluginstream_ = MediaStream("input", |
| 402 | rgbFrame->format, |
| 403 | 1 / fr, |
| 404 | rgbFrame->width, |
| 405 | rgbFrame->height, |
| 406 | 0, |
| 407 | fr); |
| 408 | |
| 409 | if (showLogo_) { |
| 410 | MediaStream markstream_ = MediaStream("mark", |
| 411 | mark_->format, |
| 412 | logoStream_.timeBase, |
| 413 | mark_->width, |
| 414 | mark_->height, |
| 415 | 0, |
| 416 | logoStream_.frameRate); |
| 417 | pluginFilter_.initialize(pluginFilterDescription_, {markstream_, pluginstream_}); |
| 418 | pluginFilter_.feedInput(mark_.get(), "mark"); |
| 419 | pluginFilter_.feedEOF("mark"); |
| 420 | } |
| 421 | |
| 422 | infosFilter_.initialize(infosDescription_, {pluginstream_}); |
| 423 | firstRun = false; |
| 424 | } |
| 425 | |
| 426 | if (!infosFilter_.initialized_ && !pluginFilter_.initialized_) |
| 427 | return; |
| 428 | |
| 429 | if (showLogo_) { |
| 430 | if (pluginFilter_.feedInput(rgbFrame.get(), "input") == 0) { |
| 431 | wmFrame filteredFrame = {pluginFilter_.readOutput(), frameFree}; |
| 432 | if (filteredFrame.get()) |
| 433 | moveFrom(rgbFrame.get(), filteredFrame.get()); |
| 434 | } |
| 435 | } |
| 436 | if (showInfos_) { |
| 437 | if (infosFilter_.feedInput(rgbFrame.get(), "input") == 0) { |
| 438 | wmFrame filteredFrame = {infosFilter_.readOutput(), frameFree}; |
| 439 | if (filteredFrame.get()) |
| 440 | moveFrom(rgbFrame.get(), filteredFrame.get()); |
| 441 | } |
| 442 | } |
| 443 | if (showInfos_ || showLogo_) { |
| 444 | moveFrom(pluginFrame, rgbFrame.get()); |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | void |
| 449 | WatermarkVideoSubscriber::attached(jami::Observable<AVFrame*>* observable) |
| 450 | { |
| 451 | Plog::log(Plog::LogPriority::INFO, TAG, "Attached!"); |
| 452 | observable_ = observable; |
| 453 | } |
| 454 | |
| 455 | void |
| 456 | WatermarkVideoSubscriber::detached(jami::Observable<AVFrame*>*) |
| 457 | { |
| 458 | pluginFilter_.clean(); |
| 459 | infosFilter_.clean(); |
| 460 | firstRun = true; |
| 461 | observable_ = nullptr; |
| 462 | Plog::log(Plog::LogPriority::INFO, TAG, "Detached!"); |
| 463 | mtx_.unlock(); |
| 464 | } |
| 465 | |
| 466 | void |
| 467 | WatermarkVideoSubscriber::detach() |
| 468 | { |
| 469 | if (observable_) { |
| 470 | mtx_.lock(); |
| 471 | firstRun = true; |
| 472 | observable_->detach(this); |
| 473 | } |
| 474 | } |
| 475 | } // namespace jami |