Enable fp16 for zipformer (#206)

* enable fp16 for lstm and zipformer models

* Fix android

android/SherpaNcnn/app/src/main/java/com/k2fsa/sherpa/ncnn/MainActivity.kt

@@ -182,7 +182,7 @@ class MainActivity : AppCompatActivity() {
             featureDim = 80
         )
         //Please change the argument "type" if you use a different model
-        val modelConfig = getModelConfig(type = 1, useGPU = false)!!
+        val modelConfig = getModelConfig(type = 2, useGPU = true)!!
         val decoderConfig = getDecoderConfig(method = "greedy_search", numActivePaths = 4)
 
         val config = RecognizerConfig(

android/SherpaNcnn/app/src/main/java/com/k2fsa/sherpa/ncnn/SherpaNcnn.kt

@@ -16,7 +16,7 @@ data class ModelConfig(
     var joinerParam: String,
     var joinerBin: String,
     var tokens: String,
-    var numThreads: Int = 4,
+    var numThreads: Int = 1,
     var useGPU: Boolean = true, // If there is a GPU and useGPU true, we will use GPU
 )
 
@@ -109,14 +109,15 @@ fun getDecoderConfig(method: String, numActivePaths: Int): DecoderConfig {
 
 /*
 @param type
-0 - https://huggingface.co/csukuangfj/sherpa-ncnn-conv-emformer-transducer-2022-12-04
+0 - https://huggingface.co/csukuangfj/sherpa-ncnn-2022-09-30
     This model supports only Chinese
 
 1 - https://huggingface.co/csukuangfj/sherpa-ncnn-conv-emformer-transducer-2022-12-06
     This model supports both English and Chinese
 
-2 - https://huggingface.co/csukuangfj/sherpa-ncnn-conv-emformer-transducer-2022-12-08
-    This is a small model with about 18 M parameters. It supports only Chinese
+2 - https://huggingface.co/csukuangfj/sherpa-ncnn-streaming-zipformer-bilingual-zh-en-2023-02-13
+    This model supports both English and Chinese
+
 
 Please follow
 https://k2-fsa.github.io/sherpa/ncnn/pretrained_models/index.html
@@ -124,6 +125,21 @@ to add more pre-trained models
  */
 fun getModelConfig(type: Int, useGPU: Boolean): ModelConfig? {
     when (type) {
+        0 -> {
+            val modelDir = "sherpa-ncnn-2022-09-30"
+            return ModelConfig(
+                encoderParam = "$modelDir/encoder_jit_trace-pnnx.ncnn.param",
+                encoderBin = "$modelDir/encoder_jit_trace-pnnx.ncnn.bin",
+                decoderParam = "$modelDir/decoder_jit_trace-pnnx.ncnn.param",
+                decoderBin = "$modelDir/decoder_jit_trace-pnnx.ncnn.bin",
+                joinerParam = "$modelDir/joiner_jit_trace-pnnx.ncnn.param",
+                joinerBin = "$modelDir/joiner_jit_trace-pnnx.ncnn.bin",
+                tokens = "$modelDir/tokens.txt",
+                numThreads = 1,
+                useGPU = useGPU,
+            )
+        }
+
         1 -> {
             val modelDir = "sherpa-ncnn-conv-emformer-transducer-2022-12-06"
             return ModelConfig(
@@ -134,22 +150,22 @@ fun getModelConfig(type: Int, useGPU: Boolean): ModelConfig? {
                 joinerParam = "$modelDir/joiner_jit_trace-pnnx.ncnn.int8.param",
                 joinerBin = "$modelDir/joiner_jit_trace-pnnx.ncnn.int8.bin",
                 tokens = "$modelDir/tokens.txt",
-                numThreads = 4,
+                numThreads = 1,
                 useGPU = useGPU,
             )
-
         }
+
         2 -> {
-            val modelDir = "sherpa-ncnn-conv-emformer-transducer-2022-12-08/v2"
+            val modelDir = "sherpa-ncnn-streaming-zipformer-bilingual-zh-en-2023-02-13"
             return ModelConfig(
-                encoderParam = "$modelDir/encoder_jit_trace-pnnx-epoch-15-avg-3.ncnn.param",
-                encoderBin = "$modelDir/encoder_jit_trace-pnnx-epoch-15-avg-3.ncnn.bin",
-                decoderParam = "$modelDir/decoder_jit_trace-pnnx-epoch-15-avg-3.ncnn.param",
-                decoderBin = "$modelDir/decoder_jit_trace-pnnx-epoch-15-avg-3.ncnn.bin",
-                joinerParam = "$modelDir/joiner_jit_trace-pnnx-epoch-15-avg-3.ncnn.param",
-                joinerBin = "$modelDir/joiner_jit_trace-pnnx-epoch-15-avg-3.ncnn.bin",
+                encoderParam = "$modelDir/encoder_jit_trace-pnnx.ncnn.param",
+                encoderBin = "$modelDir/encoder_jit_trace-pnnx.ncnn.bin",
+                decoderParam = "$modelDir/decoder_jit_trace-pnnx.ncnn.param",
+                decoderBin = "$modelDir/decoder_jit_trace-pnnx.ncnn.bin",
+                joinerParam = "$modelDir/joiner_jit_trace-pnnx.ncnn.param",
+                joinerBin = "$modelDir/joiner_jit_trace-pnnx.ncnn.bin",
                 tokens = "$modelDir/tokens.txt",
-                numThreads = 4,
+                numThreads = 1,
                 useGPU = useGPU,
             )
         }

android/SherpaNcnn/app/src/main/res/values/strings.xml

@@ -6,9 +6,6 @@
         The source code and pre-trained models are publicly available.
         Please see https://github.com/k2-fsa/sherpa-ncnn for details.
 
-        \n\n\n
-        Note: This transducer-based model supports Chinese as well as English.
-
         \n\n\n
         We use https://github.com/tencent/ncnn for neural network computation.
     </string>

sherpa-ncnn/csrc/lstm-model.cc

@@ -159,9 +159,6 @@ ncnn::Mat LstmModel::RunJoiner(ncnn::Mat &encoder_out, ncnn::Mat &decoder_out,
 
 void LstmModel::InitEncoder(const std::string &encoder_param,
                             const std::string &encoder_bin) {
-  encoder_.opt.use_packing_layout = false;
-  encoder_.opt.use_fp16_storage = false;
-
   RegisterCustomLayers(encoder_);
   InitNet(encoder_, encoder_param, encoder_bin);
 
@@ -182,9 +179,6 @@ void LstmModel::InitJoiner(const std::string &joiner_param,
 void LstmModel::InitEncoder(AAssetManager *mgr,
                             const std::string &encoder_param,
                             const std::string &encoder_bin) {
-  encoder_.opt.use_packing_layout = false;
-  encoder_.opt.use_fp16_storage = false;
-
   RegisterCustomLayers(encoder_);
   InitNet(mgr, encoder_, encoder_param, encoder_bin);
 

sherpa-ncnn/csrc/zipformer-model.cc

@@ -20,11 +20,6 @@ ZipformerModel::ZipformerModel(const ModelConfig &config) {
   decoder_.opt = config.decoder_opt;
   joiner_.opt = config.joiner_opt;
 
-  encoder_.opt.use_fp16_arithmetic = false;
-  encoder_.opt.use_fp16_storage = false;
-
-  NCNN_LOGE("Disable fp16 for Zipformer encoder");
-
   bool has_gpu = false;
 #if NCNN_VULKAN
   has_gpu = ncnn::get_gpu_count() > 0;
@@ -56,11 +51,6 @@ ZipformerModel::ZipformerModel(AAssetManager *mgr, const ModelConfig &config) {
   decoder_.opt = config.decoder_opt;
   joiner_.opt = config.joiner_opt;
 
-  encoder_.opt.use_fp16_arithmetic = false;
-  encoder_.opt.use_fp16_storage = false;
-
-  NCNN_LOGE("Disable fp16 for Zipformer encoder on Android");
-
   bool has_gpu = false;
 #if NCNN_VULKAN
   has_gpu = ncnn::get_gpu_count() > 0;