Skip to content

Commit 8f2e302

Browse files
hvt4hvt4
committed
test: add CLI test scripts for Parakeet and Whisper-Tiny ASR models
1 parent 9e40016 commit 8f2e302

2 files changed

Lines changed: 304 additions & 0 deletions

File tree

‎test_scripts/models/parakeet.sh‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
#!/bin/bash
2+
# Copyright 2026 The LiteRT CLI Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ==============================================================================
16+
17+
# LiteRT CLI Parakeet-TDT Demo & Test Script
18+
set -e
19+
20+
21+
echo -e "${BLUE}${BOLD}==================================================================${NC}"
22+
echo -e "${BLUE}${BOLD}>>> LiteRT CLI Parakeet-TDT ASR Demo Script${NC}"
23+
echo -e "${BLUE}${BOLD}==================================================================${NC}"
24+
25+
# --- Environment Setup ---
26+
export SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
27+
export REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
28+
export LITERT_CLI_ROOT="/tmp/litert_cli_parakeet_tdt"
29+
30+
# Source shared utilities
31+
source "$SCRIPT_DIR/utils.sh"
32+
33+
34+
# Clean up and create work directory
35+
echo -e "\n${YELLOW}Setting up workspace at: $LITERT_CLI_ROOT...${NC}"
36+
rm -rf "$LITERT_CLI_ROOT"
37+
mkdir -p "$LITERT_CLI_ROOT"
38+
cd "$LITERT_CLI_ROOT"
39+
40+
# Create Python virtual environment
41+
echo -e "${YELLOW}Creating Python virtual environment...${NC}"
42+
python3 -m venv venv_parakeet
43+
source venv_parakeet/bin/activate
44+
45+
# Create output directories
46+
export MODEL_DIR="$LITERT_CLI_ROOT/models"
47+
mkdir -p "$MODEL_DIR"
48+
49+
# Test data directory
50+
export TEST_DATA_DIR="$REPO_ROOT/litert_cli/test_data"
51+
52+
# Install litert-cli from source
53+
echo -e "${YELLOW}Installing litert-cli from source...${NC}"
54+
pip install -e "$REPO_ROOT"
55+
56+
# Upgrade pip and setuptools to ensure build-system requirements (like
57+
# setuptools>=61.0) can be met
58+
echo -e "${YELLOW}Upgrading pip and setuptools...${NC}"
59+
pip install --upgrade pip setuptools wheel
60+
61+
# --- 1. Download Parakeet-TDT model ---
62+
run_case "Download: Parakeet-TDT from HuggingFace" \
63+
litert download litert-community/parakeet-tdt-0.6b-v3 --file "parakeet_tdt_0.6b_v3_5s_f32.tflite" --output "$MODEL_DIR/parakeet_tdt"
64+
65+
# Verify the downloaded model exists
66+
PARAKEET_TFLITE="$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_5s_f32.tflite"
67+
if [ ! -f "$PARAKEET_TFLITE" ]; then
68+
echo -e "${RED}Error: Downloaded model not found at $PARAKEET_TFLITE${NC}"
69+
exit 1
70+
fi
71+
72+
# --- 2. Quantize the Parakeet-TDT model ---
73+
run_case "Quantize: Parakeet-TDT Dynamic Range INT8" \
74+
litert quantize "$PARAKEET_TFLITE" --type int8_dynamic --output "$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_5s_int8_dynamic.tflite"
75+
76+
run_case "Quantize: Parakeet-TDT Weight-Only INT8" \
77+
litert quantize "$PARAKEET_TFLITE" --type int8_weight_only --output "$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_5s_int8_weight_only.tflite"
78+
79+
# --- 3. Run Inference (Desktop & Android) ---
80+
run_case "Run: Parakeet-TDT FP32 on Desktop (CPU)" \
81+
litert run "$PARAKEET_TFLITE" --desktop --cpu --iterations 1
82+
83+
if has_desktop_gpu "$PARAKEET_TFLITE"; then
84+
run_case "Run: Parakeet-TDT FP32 on Desktop (GPU)" \
85+
litert run "$PARAKEET_TFLITE" --desktop --gpu --iterations 1
86+
else
87+
echo -e "\n${YELLOW}Desktop GPU delegate is not supported. Skipping Desktop GPU run.${NC}"
88+
fi
89+
90+
91+
run_case "Run: Parakeet-TDT Dynamic INT8 on Desktop (CPU)" \
92+
litert run "$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_5s_int8_dynamic.tflite" --desktop --cpu --iterations 1
93+
94+
if has_android_device; then
95+
echo -e "\n${GREEN}Android device detected. Running Android inference...${NC}"
96+
run_case "Run: Parakeet-TDT FP32 on Android (CPU)" \
97+
litert run "$PARAKEET_TFLITE" --android --cpu --iterations 1
98+
99+
run_case "Run: Parakeet-TDT FP32 on Android (GPU)" \
100+
litert run "$PARAKEET_TFLITE" --android --gpu --iterations 1
101+
102+
run_case "Run: Parakeet-TDT Dynamic INT8 on Android (CPU)" \
103+
litert run "$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_5s_int8_dynamic.tflite" --android --cpu --iterations 1
104+
fi
105+
106+
# --- 4. Benchmark (Android) ---
107+
if has_android_device; then
108+
echo -e "\n${GREEN}Android device detected. Running Android benchmarks...${NC}"
109+
run_case "Benchmark: Parakeet-TDT FP32 on Android (CPU)" \
110+
litert benchmark "$PARAKEET_TFLITE" --android
111+
112+
run_case "Benchmark: Parakeet-TDT FP32 on Android (GPU)" \
113+
litert benchmark "$PARAKEET_TFLITE" --android --gpu
114+
115+
run_case "Benchmark: Parakeet-TDT Dynamic INT8 on Android" \
116+
litert benchmark "$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_5s_int8_dynamic.tflite" --android
117+
else
118+
echo -e "\n${YELLOW}No Android device detected. Skipping benchmarks on Android.${NC}"
119+
fi
120+
121+
122+
# # --- 5. Compile (AOT Compilation) ---
123+
# run_case "Compile: Parakeet-TDT FP32 for Qualcomm sm8750 NPU" \
124+
# litert compile "$PARAKEET_TFLITE" --target sm8750 --output-dir "$MODEL_DIR/parakeet_tdt"
125+
# run_case "Compile: Parakeet-TDT FP32 for MediaTek MT6993 NPU" \
126+
# litert compile "$PARAKEET_TFLITE" --target MT6993 --output-dir "$MODEL_DIR/parakeet_tdt"
127+
128+
# # --- 6. Benchnark compiled model ---
129+
# # Enable those use cases, or change to your own targets, if you have connected those android
130+
# # devices through NPU.
131+
132+
# run_case "Run Qualcomm compiled Parakeet-TDT" \
133+
# litert run "$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_Qualcomm_SM8750.tflite" --android --npu
134+
# run_case "Benchmark Qualcomm compiled Parakeet-TDT" \
135+
# litert benchmark "$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_Qualcomm_SM8750.tflite" --android --npu
136+
137+
# run_case "Run MediaTek compiled Parakeet-TDT" \
138+
# litert run "$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_MediaTek_MT6993.tflite" --android --npu
139+
# run_case "Benchmark MediaTek compiled Parakeet-TDT" \
140+
# litert benchmark "$MODEL_DIR/parakeet_tdt/parakeet_tdt_0.6b_v3_MediaTek_MT6993.tflite" --android --npu
141+
142+
143+
# --- 7. Visualize (Model Explorer) ---
144+
run_case "Visualize: Launch Model Explorer in the background" \
145+
litert visualize "$PARAKEET_TFLITE"
146+
147+
run_case "Visualize: Stop all Model Explorer servers" \
148+
litert visualize --stop-all
149+
150+
151+
# --- Summary Report ---
152+
print_summary_report "Parakeet-TDT"

‎test_scripts/models/whisper_tiny.sh‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
#!/bin/bash
2+
# Copyright 2026 The LiteRT CLI Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ==============================================================================
16+
17+
# LiteRT CLI Whisper-Tiny Demo & Test Script
18+
set -e
19+
20+
21+
echo -e "${BLUE}${BOLD}==================================================================${NC}"
22+
echo -e "${BLUE}${BOLD}>>> LiteRT CLI Whisper-Tiny ASR Demo Script${NC}"
23+
echo -e "${BLUE}${BOLD}==================================================================${NC}"
24+
25+
# --- Environment Setup ---
26+
export SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
27+
export REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
28+
export LITERT_CLI_ROOT="/tmp/litert_cli_whisper_tiny"
29+
30+
# Source shared utilities
31+
source "$SCRIPT_DIR/utils.sh"
32+
33+
34+
# Clean up and create work directory
35+
echo -e "\n${YELLOW}Setting up workspace at: $LITERT_CLI_ROOT...${NC}"
36+
rm -rf "$LITERT_CLI_ROOT"
37+
mkdir -p "$LITERT_CLI_ROOT"
38+
cd "$LITERT_CLI_ROOT"
39+
40+
# Create Python virtual environment
41+
echo -e "${YELLOW}Creating Python virtual environment...${NC}"
42+
python3 -m venv venv_whisper_tiny
43+
source venv_whisper_tiny/bin/activate
44+
45+
# Create output directories
46+
export MODEL_DIR="$LITERT_CLI_ROOT/models"
47+
mkdir -p "$MODEL_DIR"
48+
49+
# Test data directory
50+
export TEST_DATA_DIR="$REPO_ROOT/litert_cli/test_data"
51+
52+
# Install litert-cli from source
53+
echo -e "${YELLOW}Installing litert-cli from source...${NC}"
54+
pip install -e "$REPO_ROOT"
55+
56+
# Upgrade pip and setuptools to ensure build-system requirements (like
57+
# setuptools>=61.0) can be met
58+
echo -e "${YELLOW}Upgrading pip and setuptools...${NC}"
59+
pip install --upgrade pip setuptools wheel
60+
61+
# --- 1. Download Whisper-Tiny model ---
62+
run_case "Download: Whisper-Tiny from HuggingFace" \
63+
litert download litert-community/whisper-tiny --file "whisper_tiny_30s_f32.tflite" --output "$MODEL_DIR/whisper_tiny"
64+
65+
# Verify the downloaded model exists
66+
WHISPER_TFLITE="$MODEL_DIR/whisper_tiny/whisper_tiny_30s_f32.tflite"
67+
if [ ! -f "$WHISPER_TFLITE" ]; then
68+
echo -e "${RED}Error: Downloaded model not found at $WHISPER_TFLITE${NC}"
69+
exit 1
70+
fi
71+
72+
# --- 2. Quantize the Whisper-Tiny model ---
73+
run_case "Quantize: Whisper-Tiny Dynamic Range INT8" \
74+
litert quantize "$WHISPER_TFLITE" --type int8_dynamic --output "$MODEL_DIR/whisper_tiny/whisper_tiny_30s_int8_dynamic.tflite"
75+
76+
run_case "Quantize: Whisper-Tiny Weight-Only INT8" \
77+
litert quantize "$WHISPER_TFLITE" --type int8_weight_only --output "$MODEL_DIR/whisper_tiny/whisper_tiny_30s_int8_weight_only.tflite"
78+
79+
# --- 3. Run Inference (Desktop & Android) ---
80+
run_case "Run: Whisper-Tiny FP32 on Desktop (CPU)" \
81+
litert run "$WHISPER_TFLITE" --desktop --cpu --iterations 1
82+
83+
if has_desktop_gpu "$WHISPER_TFLITE"; then
84+
run_case "Run: Whisper-Tiny FP32 on Desktop (GPU)" \
85+
litert run "$WHISPER_TFLITE" --desktop --gpu --iterations 1
86+
else
87+
echo -e "\n${YELLOW}Desktop GPU delegate is not supported. Skipping Desktop GPU run.${NC}"
88+
fi
89+
90+
91+
run_case "Run: Whisper-Tiny Dynamic INT8 on Desktop (CPU)" \
92+
litert run "$MODEL_DIR/whisper_tiny/whisper_tiny_30s_int8_dynamic.tflite" --desktop --cpu --iterations 1
93+
94+
if has_android_device; then
95+
echo -e "\n${GREEN}Android device detected. Running Android inference...${NC}"
96+
run_case "Run: Whisper-Tiny FP32 on Android (CPU)" \
97+
litert run "$WHISPER_TFLITE" --android --cpu --iterations 1
98+
99+
run_case "Run: Whisper-Tiny FP32 on Android (GPU)" \
100+
litert run "$WHISPER_TFLITE" --android --gpu --iterations 1
101+
102+
run_case "Run: Whisper-Tiny Dynamic INT8 on Android (CPU)" \
103+
litert run "$MODEL_DIR/whisper_tiny/whisper_tiny_30s_int8_dynamic.tflite" --android --cpu --iterations 1
104+
fi
105+
106+
# --- 4. Benchmark (Android) ---
107+
if has_android_device; then
108+
echo -e "\n${GREEN}Android device detected. Running Android benchmarks...${NC}"
109+
run_case "Benchmark: Whisper-Tiny FP32 on Android (CPU)" \
110+
litert benchmark "$WHISPER_TFLITE" --android
111+
112+
run_case "Benchmark: Whisper-Tiny FP32 on Android (GPU)" \
113+
litert benchmark "$WHISPER_TFLITE" --android --gpu
114+
115+
run_case "Benchmark: Whisper-Tiny Dynamic INT8 on Android" \
116+
litert benchmark "$MODEL_DIR/whisper_tiny/whisper_tiny_30s_int8_dynamic.tflite" --android
117+
else
118+
echo -e "\n${YELLOW}No Android device detected. Skipping benchmarks on Android.${NC}"
119+
fi
120+
121+
# # --- 5. Compile (AOT Compilation) ---
122+
# run_case "Compile: Whisper-Tiny FP32 for Qualcomm sm8750 NPU" \
123+
# litert compile "$WHISPER_TFLITE" --target sm8750 --output-dir "$MODEL_DIR/whisper_tiny"
124+
# run_case "Compile: Whisper-Tiny FP32 for MediaTek MT6993 NPU" \
125+
# litert compile "$WHISPER_TFLITE" --target MT6993 --output-dir "$MODEL_DIR/whisper_tiny"
126+
127+
# # --- 6. Benchnark compiled model ---
128+
# # Enable those use cases, or change to your own targets, if you have connected those android
129+
# # devices through NPU.
130+
131+
# run_case "Run Qualcomm compiled Whisper-Tiny" \
132+
# litert run "$MODEL_DIR/whisper_tiny/whisper_tiny_30s_Qualcomm_SM8750.tflite" --android --npu
133+
# run_case "Benchmark Qualcomm compiled Whisper-Tiny" \
134+
# litert benchmark "$MODEL_DIR/whisper_tiny/whisper_tiny_30s_Qualcomm_SM8750.tflite" --android --npu
135+
136+
# run_case "Run MediaTek compiled Whisper-Tiny" \
137+
# litert run "$MODEL_DIR/whisper_tiny/whisper_tiny_30s_MediaTek_MT6993.tflite" --android --npu
138+
# run_case "Benchmark MediaTek compiled Whisper-Tiny" \
139+
# litert benchmark "$MODEL_DIR/whisper_tiny/whisper_tiny_30s_MediaTek_MT6993.tflite" --android --npu
140+
141+
142+
143+
# --- 6. Visualize (Model Explorer) ---
144+
run_case "Visualize: Launch Model Explorer in the background" \
145+
litert visualize "$WHISPER_TFLITE"
146+
147+
run_case "Visualize: Stop all Model Explorer servers" \
148+
litert visualize --stop-all
149+
150+
151+
# --- Summary Report ---
152+
print_summary_report "Whisper-Tiny"

0 commit comments

Comments
 (0)