File size: 3,169 Bytes
f53e03f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | #pragma once
#include "Utility.h"
#include <string>
#include <memory>
#include <vector>
#include <map>
enum class Precision
{
FP32,
FP16
};
enum class DataType
{
FLOAT,
HALF,
INT8,
INT32,
BOOL,
UINT8,
INT64,
UNKNOWN
};
struct Options
{
using AxisNames = std::map< std::string, std::map<int, std::string> >;
using AxisSizes = std::map<std::string, std::tuple<int, int, int> >;
using ShapeTensorSizes = std::map<std::string, std::tuple<std::vector<int>, std::vector<int>, std::vector<int>> >;
Precision precision = Precision::FP32;
AxisNames dynamic_axes_names;
AxisSizes dynamic_axes_sizes;
ShapeTensorSizes shape_tensor_sizes;
std::tuple<int, int, int> defaultSizes = { 1,8,16 };
int deviceID = 0;
};
bool ConvertONNXToTRT(
const Options& options,
const std::string& onnxModelPath,
std::string& generatedTRTFile,
const std::string prefix = "",
bool forceConvert = false
);
// Forward declaration
typedef struct CUstream_st *cudaStream_t;
class TRTInferenceEngine
{
public:
TRTInferenceEngine();
~TRTInferenceEngine();
using AxisSizes = std::map<std::string, int >;
bool Initialize(const std::string& trtPath, int deviceID, const Options::AxisNames& axisNames = {});
bool InitInputs(const AxisSizes& axisSizes = {});
void Destroy();
void SetInputData(const std::string& name, const void* data, size_t byteCount);
template<typename T> void SetInputData(const std::string& name, const T* data, size_t elementCount);
template<typename T> void SetInputData(const std::string& name, const TPinnedVector<T>& data);
void GetOutputData(const std::string& name, void* data, size_t byteCount);
template<typename T> void GetOutputData(const std::string& name, T* data, size_t elementCount);
template<typename T> void GetOutputData(const std::string& name, TPinnedVector<T>& data);
void SetInputDataAsync(const std::string& name, const void* data, size_t byteCount, cudaStream_t stream);
template<typename T> void SetInputDataAsync(const std::string& name, const T* data, size_t elementCount, cudaStream_t stream);
template<typename T> void SetInputDataAsync(const std::string& name, const TPinnedVector<T>& data, cudaStream_t stream);
void GetOutputDataAsync(const std::string& name, void* data, size_t byteCount, cudaStream_t stream);
template<typename T> void GetOutputDataAsync(const std::string& name, T* data, size_t elementCount, cudaStream_t stream);
template<typename T> void GetOutputDataAsync(const std::string& name, TPinnedVector<T>& data, cudaStream_t stream);
std::vector<std::string> GetInputTensorNames() const;
std::vector<std::string> GetOutputTensorNames() const;
bool GetTensorShape(std::string name, std::vector<int64_t>& shape) const;
DataType GetTensorDataType(std::string name) const;
bool Enqueue(cudaStream_t stream);
private:
class Impl;
std::shared_ptr<Impl> m_impl = nullptr;
};
#include "InferenceEngine.inl"
|