GreenScreen/pluginProcessor.h

/**
 *  Copyright (C) 2020-2021 Savoir-faire Linux Inc.
 *
 *  Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
 * USA.
 */

#pragma once
// STL
#include <condition_variable>
#include <cstdint>
#include <map>
#include <memory>
#include <mutex>
#include <thread>
#include <vector>
// AvFrame
extern "C" {
#include <libavutil/frame.h>
}
// Plugin
#include <plugin/jamiplugin.h>
#include <plugin/mediahandler.h>

// Opencv processing
#include <opencv2/core.hpp>
#include <onnxruntime_cxx_api.h>
#ifdef NVIDIA
#include <cuda_provider_factory.h>
#endif
#ifdef ANDROID
#include <nnapi_provider_factory.h>
#endif
namespace jami {

class PluginProcessor
{
public:
    PluginProcessor(const std::string& dataPath, const std::string& model, const std::string& backgroundImage, bool acc);
    ~PluginProcessor();

    void initModel(const std::string& modelPath);
    /**
     * @brief feedInput
     * Takes a frame and feeds it to the model storage for predictions
     * @param frame
     */
    void feedInput(const cv::Mat& frame);

    /**
     * @brief computePredictions
     * Uses the model to compute the predictions and store them in
     * computedPredictions
     */
    void computePredictions();

    void printMask();
    void drawMaskOnFrame(cv::Mat& frame,
                         cv::Mat& frameReduced,
                         std::vector<float> computedMask,
                         int lineSize,
                         int angle);
    int getBackgroundRotation();
    void setBackgroundRotation(int angle);
    void setBackgroundImage(const std::string& backgroundPath);
    void rotateFrame(int angle, cv::Mat& mat);
    bool hasBackground() const;
    void resetInitValues(const cv::Size& modelInputSize);
    bool isAllocated();

    // Output predictions
    std::vector<float> computedMask;

    cv::Mat previousMasks[2];
    cv::Mat backgroundImage;

    cv::Size kSize;

    std::string backgroundPath;

private:
    int count{0};
    cv::Mat frame;
    int backgroundRotation{0};
    bool hasBackground_{false};
    cv::Mat bgdModel, fgdModel;
    int grabCutMode{1}; // cv::GC_INIT_WITH_MASK = 1;
    int grabCutIterations{5};
    int grabcutClass{3};
    int frameCount{3};
    float smoothFactors[3] = {0.6f, 0.3f, 0.1f};
    float kernelSize{0.05f};

    bool isAllocated_{false};
    Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "test"};
    Ort::Value input_tensor_{nullptr};
    std::array<int64_t, 3> input_shape_{257, 257, 3};

    Ort::Value output_tensor_{nullptr};
    std::array<int64_t, 4> output_shape_{1, 17, 17, 1};


    std::array<float, 257 * 257 * 3> input_image_{};
    
    std::array<float, 17 * 17> results_{};
    Ort::Session* session_{};
    const char* input_names[8] = {"image:0"};
    const char* output_names[11] = {"Identity:0"};
    Ort::SessionOptions sessOpt_;

    bool activateAcc_{false};
};
} // namespace jami