Skip to content

fix the inconsistency issue in exporting torchscript models#582

Open
wenhyan wants to merge 1 commit into
RangiLyu:mainfrom
wenhyan:bugfix#581
Open

fix the inconsistency issue in exporting torchscript models#582
wenhyan wants to merge 1 commit into
RangiLyu:mainfrom
wenhyan:bugfix#581

Conversation

@wenhyan

@wenhyan wenhyan commented May 29, 2025

Copy link
Copy Markdown
  • onnx 对比脚本
import numpy as np
import torch

def test_onnx_inference(onnx_path, input_shape=(1, 3, 416, 416)):
    session = ort.InferenceSession(onnx_path, providers=['CPUExecutionProvider'])

    input_name = session.get_inputs()[0].name
    output_names = [o.name for o in session.get_outputs()]

    torch.manual_seed(0)
    v_0 = torch.rand(1, 3, 416, 416, dtype=torch.float)
    v_0 = torch.tensor(v_0)

    input_tensor_np = v_0.numpy()

    print(f"[INFO] Input tensor shape: {input_tensor_np.shape}")

    outputs = session.run(output_names, {input_name: input_tensor_np})

    return torch.tensor(outputs[0])

if __name__ == "__main__":
    onnx_model_path = "nanodet.onnx"  # 使用 export_onnx.py 导出的模型 
    print(test_onnx_inference(onnx_model_path)[0][0])
  • torchscript 对比脚本
import torch
import numpy as np


def test_inference(use_raw=False, raw_path=None):
    net = torch.jit.load("nanodet.torchscript.pth") # 使用export_torchscript.py 导出的模型
    # net = torch.jit.load("nanodet.torchscript_with_sigmoid.pth") # 使用export_torchscript.py 导出的模型
    net.eval()

    torch.manual_seed(0)
    v_0 = torch.rand(1, 3, 416, 416, dtype=torch.float)

    with torch.no_grad():
        output = net(v_0)

    if isinstance(output, (list, tuple)):
        for i, out in enumerate(output):
            print(f"Output {i}: shape = {out.shape}")
    else:
        print("Output shape:", output.shape)
    return output

if __name__ == "__main__":
    print(test_inference()[0][0])  #


这里脚本推理的结果可以看到，分类预测的80个值是不一致的，因为onnx加过sigmoid，输出的是概率。

$ python3 nanodet_plus_m_416.torchscript_onnx.py 
nanodet_plus_m_416.torchscript_onnx.py:13: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  v_0 = torch.tensor(v_0)
[INFO] Input tensor shape: (1, 3, 416, 416)
tensor([ 1.0161e-02,  3.7889e-03,  5.4802e-03,  1.8775e-03,  1.7585e-03,
         2.1909e-03,  1.8171e-03,  3.1141e-03,  2.7342e-03,  3.6772e-03,
         1.6088e-03,  2.5381e-03,  1.9768e-03,  4.6172e-03,  3.6783e-03,
         3.0748e-03,  2.4812e-03,  4.2174e-03,  2.3261e-03,  2.0368e-03,
         1.7409e-03,  2.1859e-03,  1.8695e-03,  2.5565e-03,  1.7709e-03,
         3.7471e-03,  2.0130e-03,  3.0714e-03,  2.4747e-03,  1.4608e-03,
         2.2231e-03,  1.0444e-03,  2.8534e-03,  3.3432e-03,  2.9627e-03,
         9.9540e-04,  1.9516e-03,  2.0765e-03,  3.3031e-03,  5.5228e-03,
         6.0802e-03,  4.9037e-03,  3.0450e-03,  3.9238e-03,  4.2158e-03,
         3.6744e-03,  1.7935e-03,  2.3090e-03,  8.6510e-04,  2.3798e-03,
         2.0960e-03,  2.6433e-03,  9.9409e-04,  3.0464e-03,  1.7930e-03,
         2.1650e-03,  6.7869e-03,  4.1447e-03,  7.1998e-03,  3.0954e-03,
         6.1073e-03,  2.7908e-03,  5.0793e-03,  3.1500e-03,  1.7223e-03,
         3.3424e-03,  3.0479e-03,  1.9525e-03,  4.0565e-03,  4.3103e-03,
         1.1329e-03,  4.6557e-03,  5.6523e-03,  5.5580e-03,  6.8101e-03,
         5.4017e-03,  2.8081e-03,  3.4465e-03,  1.9233e-03,  2.3803e-03,
         1.4547e+00, -1.4144e+00, -2.5380e+00, -3.3361e+00, -2.5723e+00,
        -2.8061e+00, -2.5385e+00, -8.3413e-01,  1.9817e+00, -1.1483e+00,
        -1.7959e+00, -2.1441e+00, -2.5856e+00, -2.3415e+00, -2.3571e+00,
        -1.0275e+00,  1.7492e-01, -2.5458e-01, -1.1379e+00, -1.8016e+00,
        -1.9051e+00, -2.0250e+00, -1.9100e+00,  5.2750e-01, -7.8754e-01,
        -9.8125e-01, -2.0321e+00, -1.8283e+00, -2.0533e+00, -1.7858e+00,
        -8.8011e-01,  1.6462e+00])
$ python3 nanodet_plus_m_416.torchscript_torchscript.py 
Output shape: torch.Size([1, 3598, 112])
tensor([-4.5789, -5.5719, -5.2011, -6.2759, -6.3416, -6.1212, -6.3087, -5.7687,
        -5.8992, -5.6019, -6.4307, -5.9738, -6.2243, -5.3733, -5.6016, -5.7814,
        -5.9965, -5.4643, -6.0612, -6.1943, -6.3516, -6.1236, -6.2802, -5.9666,
        -6.3345, -5.5830, -6.2061, -5.7825, -5.9992, -6.5273, -6.1066, -6.8632,
        -5.8564, -5.6975, -5.8187, -6.9114, -6.2371, -6.1750, -5.7096, -5.1933,
        -5.0966, -5.3129, -5.7912, -5.5368, -5.4647, -5.6027, -6.3218, -6.0687,
        -7.0518, -6.0384, -6.1656, -5.9331, -6.9127, -5.7907, -6.3221, -6.1332,
        -4.9860, -5.4818, -4.9265, -5.7748, -5.0921, -5.8786, -5.2775, -5.7572,
        -6.3624, -5.6977, -5.7903, -6.2367, -5.5034, -5.4424, -6.7819, -5.3650,
        -5.1700, -5.1869, -4.9825, -5.2156, -5.8724, -5.6669, -6.2518, -6.0381,
         1.4547, -1.4144, -2.5380, -3.3361, -2.5723, -2.8061, -2.5385, -0.8341,
         1.9817, -1.1483, -1.7959, -2.1441, -2.5856, -2.3415, -2.3571, -1.0275,
         0.1749, -0.2546, -1.1379, -1.8016, -1.9051, -2.0250, -1.9100,  0.5275,
        -0.7875, -0.9812, -2.0321, -1.8283, -2.0533, -1.7858, -0.8801,  1.6462])

修复torchscript导出脚本后的结果，导出的torchscript有了sigmoid，结果一致。

$ python3 nanodet_plus_m_416.torchscript_onnx.py 
nanodet_plus_m_416.torchscript_onnx.py:13: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  v_0 = torch.tensor(v_0)
[INFO] Input tensor shape: (1, 3, 416, 416)
tensor([ 1.0161e-02,  3.7889e-03,  5.4802e-03,  1.8775e-03,  1.7585e-03,
         2.1909e-03,  1.8171e-03,  3.1141e-03,  2.7342e-03,  3.6772e-03,
         1.6088e-03,  2.5381e-03,  1.9768e-03,  4.6172e-03,  3.6783e-03,
         3.0748e-03,  2.4812e-03,  4.2174e-03,  2.3261e-03,  2.0368e-03,
         1.7409e-03,  2.1859e-03,  1.8695e-03,  2.5565e-03,  1.7709e-03,
         3.7471e-03,  2.0130e-03,  3.0714e-03,  2.4747e-03,  1.4608e-03,
         2.2231e-03,  1.0444e-03,  2.8534e-03,  3.3432e-03,  2.9627e-03,
         9.9540e-04,  1.9516e-03,  2.0765e-03,  3.3031e-03,  5.5228e-03,
         6.0802e-03,  4.9037e-03,  3.0450e-03,  3.9238e-03,  4.2158e-03,
         3.6744e-03,  1.7935e-03,  2.3090e-03,  8.6510e-04,  2.3798e-03,
         2.0960e-03,  2.6433e-03,  9.9409e-04,  3.0464e-03,  1.7930e-03,
         2.1650e-03,  6.7869e-03,  4.1447e-03,  7.1998e-03,  3.0954e-03,
         6.1073e-03,  2.7908e-03,  5.0793e-03,  3.1500e-03,  1.7223e-03,
         3.3424e-03,  3.0479e-03,  1.9525e-03,  4.0565e-03,  4.3103e-03,
         1.1329e-03,  4.6557e-03,  5.6523e-03,  5.5580e-03,  6.8101e-03,
         5.4017e-03,  2.8081e-03,  3.4465e-03,  1.9233e-03,  2.3803e-03,
         1.4547e+00, -1.4144e+00, -2.5380e+00, -3.3361e+00, -2.5723e+00,
        -2.8061e+00, -2.5385e+00, -8.3413e-01,  1.9817e+00, -1.1483e+00,
        -1.7959e+00, -2.1441e+00, -2.5856e+00, -2.3415e+00, -2.3571e+00,
        -1.0275e+00,  1.7492e-01, -2.5458e-01, -1.1379e+00, -1.8016e+00,
        -1.9051e+00, -2.0250e+00, -1.9100e+00,  5.2750e-01, -7.8754e-01,
        -9.8125e-01, -2.0321e+00, -1.8283e+00, -2.0533e+00, -1.7858e+00,
        -8.8011e-01,  1.6462e+00])
$ python3 nanodet_plus_m_416.torchscript_torchscript.py 
Output shape: torch.Size([1, 3598, 112])
tensor([ 1.0161e-02,  3.7889e-03,  5.4801e-03,  1.8775e-03,  1.7584e-03,
         2.1910e-03,  1.8171e-03,  3.1141e-03,  2.7343e-03,  3.6773e-03,
         1.6088e-03,  2.5380e-03,  1.9768e-03,  4.6172e-03,  3.6783e-03,
         3.0748e-03,  2.4812e-03,  4.2175e-03,  2.3262e-03,  2.0368e-03,
         1.7409e-03,  2.1859e-03,  1.8695e-03,  2.5565e-03,  1.7708e-03,
         3.7471e-03,  2.0130e-03,  3.0714e-03,  2.4747e-03,  1.4608e-03,
         2.2231e-03,  1.0444e-03,  2.8533e-03,  3.3432e-03,  2.9627e-03,
         9.9541e-04,  1.9516e-03,  2.0765e-03,  3.3031e-03,  5.5228e-03,
         6.0801e-03,  4.9037e-03,  3.0450e-03,  3.9238e-03,  4.2157e-03,
         3.6744e-03,  1.7935e-03,  2.3089e-03,  8.6508e-04,  2.3798e-03,
         2.0961e-03,  2.6433e-03,  9.9410e-04,  3.0464e-03,  1.7930e-03,
         2.1650e-03,  6.7869e-03,  4.1447e-03,  7.1998e-03,  3.0953e-03,
         6.1073e-03,  2.7908e-03,  5.0794e-03,  3.1499e-03,  1.7223e-03,
         3.3424e-03,  3.0479e-03,  1.9526e-03,  4.0565e-03,  4.3104e-03,
         1.1329e-03,  4.6557e-03,  5.6523e-03,  5.5581e-03,  6.8101e-03,
         5.4017e-03,  2.8082e-03,  3.4465e-03,  1.9233e-03,  2.3803e-03,
         1.4547e+00, -1.4144e+00, -2.5380e+00, -3.3361e+00, -2.5723e+00,
        -2.8061e+00, -2.5385e+00, -8.3413e-01,  1.9817e+00, -1.1483e+00,
        -1.7959e+00, -2.1441e+00, -2.5856e+00, -2.3415e+00, -2.3571e+00,
        -1.0275e+00,  1.7492e-01, -2.5458e-01, -1.1379e+00, -1.8016e+00,
        -1.9051e+00, -2.0250e+00, -1.9100e+00,  5.2750e-01, -7.8754e-01,
        -9.8125e-01, -2.0321e+00, -1.8283e+00, -2.0533e+00, -1.7858e+00,
        -8.8011e-01,  1.6462e+00])

@Charles-Zhong

Charles-Zhong commented Jun 10, 2025 •
edited
Loading

Copy link
Copy Markdown

这是常规做法，减少部署时做后处理的时间

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

No reviews

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

2 participants

@wenhyan @Charles-Zhong