agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 1 | /** |
Sébastien Blin | cb783e3 | 2021-02-12 11:34:10 -0500 | [diff] [blame] | 2 | * Copyright (C) 2020-2021 Savoir-faire Linux Inc. |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 3 | * |
| 4 | * Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License as published by |
| 8 | * the Free Software Foundation; either version 3 of the License, or |
| 9 | * (at your option) any later version. |
| 10 | * |
| 11 | * This program is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | * GNU General Public License for more details. |
| 15 | * |
| 16 | * You should have received a copy of the GNU General Public License |
| 17 | * along with this program; if not, write to the Free Software |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 |
| 19 | * USA. |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 20 | */ |
| 21 | |
| 22 | #pragma once |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 23 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 24 | #include <map> |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 25 | #include <vector> |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 26 | extern "C" { |
| 27 | #include <libavutil/frame.h> |
| 28 | } |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 29 | |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 30 | #include <plugin/jamiplugin.h> |
| 31 | #include <plugin/mediahandler.h> |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 32 | |
agsantos | 796b5af | 2020-12-22 19:38:27 -0500 | [diff] [blame] | 33 | #include <opencv2/core.hpp> |
| 34 | #include <onnxruntime_cxx_api.h> |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 35 | #ifdef __ANDROID__ |
agsantos | 796b5af | 2020-12-22 19:38:27 -0500 | [diff] [blame] | 36 | #include <nnapi_provider_factory.h> |
| 37 | #endif |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 38 | |
| 39 | #include <frameFilter.h> |
| 40 | #include <frameUtils.h> |
| 41 | #include <mediaStream.h> |
| 42 | #include <functional> |
| 43 | |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 44 | namespace jami { |
| 45 | |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 46 | static const char* modelInputNames[8] = {"image:0"}; |
| 47 | static const char* modelOutputNames[11] = {"Identity:0"}; |
| 48 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 49 | class PluginProcessor |
| 50 | { |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 51 | public: |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 52 | PluginProcessor(const std::string& model, bool acc); |
agsantos | 796b5af | 2020-12-22 19:38:27 -0500 | [diff] [blame] | 53 | ~PluginProcessor(); |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 54 | |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 55 | void initModel(const std::string& modelPath, bool activateAcc); |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 56 | /** |
| 57 | * @brief feedInput |
| 58 | * Takes a frame and feeds it to the model storage for predictions |
| 59 | * @param frame |
| 60 | */ |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 61 | void feedInput(AVFrame* input); |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 62 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 63 | /** |
| 64 | * @brief computePredictions |
| 65 | * Uses the model to compute the predictions and store them in |
| 66 | * computedPredictions |
| 67 | */ |
| 68 | void computePredictions(); |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 69 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 70 | void printMask(); |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 71 | void drawMaskOnFrame(AVFrame* frame, AVFrame* frameReduced, int angle); |
| 72 | bool isAllocated() { return isAllocated_; } |
| 73 | void setBlur(bool isBlur) { isBlur_ = isBlur; } |
| 74 | void setBlurLevel(const std::string& blurLevel) { blurLevel_ = blurLevel; } |
| 75 | void setBackgroundImage(const std::string& backgroundPath) { backgroundPath_ = backgroundPath; } |
| 76 | void initFilters(const std::pair<int, int>& inputSize, int format, int angle); |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 77 | |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 78 | std::pair<int, int> modelInputDimensions {257, 257}; |
| 79 | std::map<int, std::string> rotation = {{90, "-PI/2"}, |
| 80 | {-90, "PI/2"}, |
| 81 | {-180, "-PI"}, |
| 82 | {180, "PI"}, |
| 83 | {0, "0"}}; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 84 | |
| 85 | private: |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 86 | void resetInitValues(); |
| 87 | void loadBackground(); |
| 88 | MediaStream getbgAVFrameInfos(); |
agsantos | 796b5af | 2020-12-22 19:38:27 -0500 | [diff] [blame] | 89 | |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 90 | bool isBlur_ {false}; |
| 91 | std::string blurLevel_; |
| 92 | std::string backgroundPath_; |
| 93 | cv::Mat previousMasks_[2]; |
| 94 | std::vector<float> computedMask_; |
| 95 | cv::Mat cvFrame_; |
agsantos | 796b5af | 2020-12-22 19:38:27 -0500 | [diff] [blame] | 96 | |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 97 | // process variables |
| 98 | cv::Size kSize_; |
| 99 | int count_ {0}; |
| 100 | cv::Mat bgdModel_, fgdModel_; |
| 101 | int grabCutMode_ {1}; // cv::GC_INIT_WITH_MASK = 1; |
| 102 | int grabCutIterations_ {4}; |
| 103 | int grabcutClass_ {3}; |
| 104 | int frameCount_ {5}; |
| 105 | float smoothFactors_[3] = {0.6f, 0.3f, 0.1f}; |
| 106 | float kernelSize_ {0.05f}; |
agsantos | 796b5af | 2020-12-22 19:38:27 -0500 | [diff] [blame] | 107 | |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 108 | // filters |
| 109 | std::string mainFilterDescription_; |
| 110 | FrameFilter mainFilter_; |
| 111 | std::unique_ptr<AVFormatContext, std::function<void(AVFormatContext*)>> pFormatCtx_ |
| 112 | = {avformat_alloc_context(), [](AVFormatContext* ptr) { |
| 113 | avformat_close_input(&ptr); |
| 114 | avformat_free_context(ptr); |
| 115 | }}; |
| 116 | int videoStream_ {-1}; |
| 117 | MediaStream ims_, ims2_, maskms_; |
agsantos | 796b5af | 2020-12-22 19:38:27 -0500 | [diff] [blame] | 118 | |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 119 | // onnx related |
| 120 | bool isAllocated_ {false}; |
| 121 | Ort::Env env_ {ORT_LOGGING_LEVEL_WARNING, "test"}; |
| 122 | Ort::Session* session_ {}; |
agsantos | 796b5af | 2020-12-22 19:38:27 -0500 | [diff] [blame] | 123 | Ort::SessionOptions sessOpt_; |
| 124 | |
agsantos | dd6a62a | 2021-03-29 17:13:27 -0400 | [diff] [blame] | 125 | Ort::Value input_tensor_ {nullptr}; |
| 126 | std::array<int64_t, 3> input_shape_ {257, 257, 3}; |
| 127 | std::array<float, 257 * 257 * 3> input_image_ {}; |
| 128 | |
| 129 | Ort::Value output_tensor_ {nullptr}; |
| 130 | std::array<int64_t, 4> output_shape_ {1, 17, 17, 1}; |
| 131 | std::array<float, 17 * 17> results_ {}; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 132 | }; |
| 133 | } // namespace jami |