agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 1 | /** |
Sébastien Blin | cb783e3 | 2021-02-12 11:34:10 -0500 | [diff] [blame] | 2 | * Copyright (C) 2020-2021 Savoir-faire Linux Inc. |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 3 | * |
| 4 | * Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License as published by |
| 8 | * the Free Software Foundation; either version 3 of the License, or |
| 9 | * (at your option) any later version. |
| 10 | * |
| 11 | * This program is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | * GNU General Public License for more details. |
| 15 | * |
| 16 | * You should have received a copy of the GNU General Public License |
| 17 | * along with this program; if not, write to the Free Software |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 |
| 19 | * USA. |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 20 | */ |
| 21 | |
| 22 | #include "pluginInference.h" |
| 23 | // Std libraries |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 24 | #include "pluglog.h" |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 25 | #include <cstring> |
| 26 | #include <numeric> |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 27 | |
| 28 | const char sep = separator(); |
| 29 | const std::string TAG = "FORESEG"; |
| 30 | |
| 31 | namespace jami { |
| 32 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 33 | PluginInference::PluginInference(TFModel model) |
| 34 | : TensorflowInference(model) |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 35 | { |
| 36 | #ifndef TFLITE |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 37 | // Initialize TENSORFLOW_CC lib |
| 38 | static const char* kFakeName = "fake program name"; |
| 39 | int argc = 1; |
| 40 | char* fake_name_copy = strdup(kFakeName); |
| 41 | char** argv = &fake_name_copy; |
| 42 | tensorflow::port::InitMain(kFakeName, &argc, &argv); |
| 43 | if (argc > 1) { |
| 44 | Plog::log(Plog::LogPriority::INFO, "TENSORFLOW INIT", "Unknown argument "); |
| 45 | } |
| 46 | free(fake_name_copy); |
| 47 | #endif // TFLITE |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 48 | } |
| 49 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 50 | PluginInference::~PluginInference() {} |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 51 | |
| 52 | #ifdef TFLITE |
| 53 | std::pair<uint8_t*, std::vector<int>> |
| 54 | PluginInference::getInput() |
| 55 | { |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 56 | // We assume that we have only one input |
| 57 | // Get the input index |
| 58 | int input = interpreter->inputs()[0]; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 59 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 60 | uint8_t* inputDataPointer = interpreter->typed_tensor<uint8_t>(input); |
| 61 | // Get the input dimensions vector |
| 62 | std::vector<int> dims = getTensorDimensions(input); |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 63 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 64 | return std::make_pair(inputDataPointer, dims); |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 65 | } |
| 66 | |
| 67 | // // Types returned by tensorflow |
| 68 | // int type = interpreter->tensor(outputIndex)->type |
| 69 | // typedef enum { |
| 70 | // kTfLiteNoType = 0, |
| 71 | // kTfLiteFloat32 = 1, float |
| 72 | // kTfLiteInt32 = 2, int // int32_t |
| 73 | // kTfLiteUInt8 = 3, uint8_t |
| 74 | // kTfLiteInt64 = 4, int64_t |
| 75 | // kTfLiteString = 5, |
| 76 | // kTfLiteBool = 6, |
| 77 | // kTfLiteInt16 = 7, int16_t |
| 78 | // kTfLiteComplex64 = 8, |
| 79 | // kTfLiteInt8 = 9, int8_t |
| 80 | // kTfLiteFloat16 = 10, float16_t |
| 81 | // } TfLiteType; |
| 82 | |
| 83 | std::vector<float> |
| 84 | PluginInference::masksPredictions() const |
| 85 | { |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 86 | int outputIndex = interpreter->outputs()[0]; |
| 87 | std::vector<int> dims = getTensorDimensions(outputIndex); |
| 88 | int totalDimensions = 1; |
| 89 | for (size_t i = 0; i < dims.size(); i++) { |
| 90 | totalDimensions *= dims[i]; |
| 91 | } |
| 92 | std::vector<float> out; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 93 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 94 | int type = interpreter->tensor(outputIndex)->type; |
| 95 | switch (type) { |
| 96 | case 1: { |
| 97 | float* outputDataPointer = interpreter->typed_tensor<float>(outputIndex); |
| 98 | std::vector<float> output(outputDataPointer, outputDataPointer + totalDimensions); |
| 99 | out = std::vector<float>(output.begin(), output.end()); |
| 100 | break; |
| 101 | } |
| 102 | case 2: { |
| 103 | int* outputDataPointer = interpreter->typed_tensor<int>(outputIndex); |
| 104 | std::vector<int> output(outputDataPointer, outputDataPointer + totalDimensions); |
| 105 | out = std::vector<float>(output.begin(), output.end()); |
| 106 | break; |
| 107 | } |
| 108 | case 4: { |
| 109 | int64_t* outputDataPointer = interpreter->typed_tensor<int64_t>(outputIndex); |
| 110 | std::vector<int64_t> output(outputDataPointer, outputDataPointer + totalDimensions); |
| 111 | out = std::vector<float>(output.begin(), output.end()); |
| 112 | break; |
| 113 | } |
| 114 | } |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 115 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 116 | return out; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 117 | } |
| 118 | |
| 119 | void |
| 120 | PluginInference::setExpectedImageDimensions() |
| 121 | { |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 122 | // We assume that we have only one input |
| 123 | // Get the input index |
| 124 | int input = interpreter->inputs()[0]; |
| 125 | // Get the input dimensions vector |
| 126 | std::vector<int> dims = getTensorDimensions(input); |
| 127 | |
| 128 | imageWidth = dims.at(1); |
| 129 | imageHeight = dims.at(2); |
| 130 | imageNbChannels = dims.at(3); |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 131 | } |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 132 | #else // TFLITE |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 133 | // Given an image file name, read in the data, try to decode it as an image, |
| 134 | // resize it to the requested size, and then scale the values as desired. |
| 135 | void |
| 136 | PluginInference::ReadTensorFromMat(const cv::Mat& image) |
| 137 | { |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 138 | imageTensor = tensorflow::Tensor(tensorflow::DataType::DT_FLOAT, |
| 139 | tensorflow::TensorShape({1, image.cols, image.rows, 3})); |
| 140 | float* p = imageTensor.flat<float>().data(); |
| 141 | cv::Mat temp(image.rows, image.cols, CV_32FC3, p); |
| 142 | image.convertTo(temp, CV_32FC3); |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 143 | } |
| 144 | |
| 145 | std::vector<float> |
| 146 | PluginInference::masksPredictions() const |
| 147 | { |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 148 | std::vector<int> dims; |
| 149 | int flatSize = 1; |
| 150 | int num_dimensions = outputs[0].shape().dims(); |
| 151 | for (int ii_dim = 0; ii_dim < num_dimensions; ii_dim++) { |
| 152 | dims.push_back(outputs[0].shape().dim_size(ii_dim)); |
| 153 | flatSize *= outputs[0].shape().dim_size(ii_dim); |
| 154 | } |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 155 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 156 | std::vector<float> out; |
| 157 | int type = outputs[0].dtype(); |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 158 | |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 159 | switch (type) { |
| 160 | case tensorflow::DataType::DT_FLOAT: { |
| 161 | for (int offset = 0; offset < flatSize; offset++) { |
| 162 | out.push_back(outputs[0].flat<float>()(offset)); |
| 163 | } |
| 164 | break; |
| 165 | } |
| 166 | case tensorflow::DataType::DT_INT32: { |
| 167 | for (int offset = 0; offset < flatSize; offset++) { |
| 168 | out.push_back(static_cast<float>(outputs[0].flat<tensorflow::int32>()(offset))); |
| 169 | } |
| 170 | break; |
| 171 | } |
| 172 | case tensorflow::DataType::DT_INT64: { |
| 173 | for (int offset = 0; offset < flatSize; offset++) { |
| 174 | out.push_back(static_cast<float>(outputs[0].flat<tensorflow::int64>()(offset))); |
| 175 | } |
| 176 | break; |
| 177 | } |
| 178 | default: { |
| 179 | for (int offset = 0; offset < flatSize; offset++) { |
| 180 | out.push_back(0); |
| 181 | } |
| 182 | break; |
| 183 | } |
| 184 | } |
| 185 | return out; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 186 | } |
| 187 | |
| 188 | void |
| 189 | PluginInference::setExpectedImageDimensions() |
| 190 | { |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 191 | if (tfModel.dims[1] != 0) |
| 192 | imageWidth = tfModel.dims[1]; |
| 193 | if (tfModel.dims[2] != 0) |
| 194 | imageHeight = tfModel.dims[2]; |
| 195 | if (tfModel.dims[3] != 0) |
| 196 | imageNbChannels = tfModel.dims[3]; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 197 | } |
| 198 | #endif |
| 199 | |
| 200 | int |
| 201 | PluginInference::getImageWidth() const |
| 202 | { |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 203 | return imageWidth; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 204 | } |
| 205 | |
| 206 | int |
| 207 | PluginInference::getImageHeight() const |
| 208 | { |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 209 | return imageHeight; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 210 | } |
| 211 | |
| 212 | int |
| 213 | PluginInference::getImageNbChannels() const |
| 214 | { |
agsantos | ac1940d | 2020-09-17 10:18:40 -0400 | [diff] [blame] | 215 | return imageNbChannels; |
agsantos | 5aa3965 | 2020-08-11 18:18:04 -0400 | [diff] [blame] | 216 | } |
| 217 | } // namespace jami |