update to the latest master of ncnn (#43)

build-android-arm64-v8a-with-vulkan.sh

@@ -120,4 +120,5 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake"
 make VERBOSE=1 -j4
 make install/strip
 
-cp -v my-glslang/build/install/lib/lib*.so install/lib/
+cp -v my-glslang/build/install/lib/libSPIRV.so install/lib/
+cp -v my-glslang/build/install/lib/libglslang.so install/lib/

cmake/ncnn.cmake

@@ -7,10 +7,10 @@ function(download_ncnn)
 
   # If you don't have access to the internet, please download it to your
   # local drive and modify the following line according to your needs.
-  # set(ncnn_URL  "file:///star-fj/fangjun/download/github/ncnn-sherpa-0.7.tar.gz")
-  set(ncnn_URL "https://github.com/csukuangfj/ncnn/archive/refs/tags/sherpa-0.7.tar.gz")
+  # set(ncnn_URL  "file:///star-fj/fangjun/download/github/ncnn-sherpa-0.8.tar.gz")
+  set(ncnn_URL "https://github.com/csukuangfj/ncnn/archive/refs/tags/sherpa-0.8.tar.gz")
 
-  set(ncnn_HASH "SHA256=fdf3cc29a43bfb3e2d7cdbbc98a7e69d0a3cc8922b67c47c4c2c8ac28125ae9c")
+  set(ncnn_HASH "SHA256=f605c48986406800615d00cf14b955e95f73286eadacedb6c3371542540e1df0")
 
   FetchContent_Declare(ncnn
     URL               ${ncnn_URL}
@@ -110,7 +110,7 @@ function(download_ncnn)
     Mish
     StatisticsPooling
     Swish
-    Gemm
+    # Gemm
     GroupNorm
     LayerNorm
     Softplus

sherpa-ncnn/csrc/conv-emformer-model.cc

@@ -15,8 +15,7 @@
 
 namespace sherpa_ncnn {
 
-ConvEmformerModel::ConvEmformerModel(const ModelConfig &config)
-    : num_threads_(config.num_threads) {
+ConvEmformerModel::ConvEmformerModel(const ModelConfig &config) {
   encoder_.opt = config.encoder_opt;
   decoder_.opt = config.decoder_opt;
   joiner_.opt = config.joiner_opt;
@@ -48,8 +47,7 @@ ConvEmformerModel::ConvEmformerModel(const ModelConfig &config)
 
 #if __ANDROID_API__ >= 9
 ConvEmformerModel::ConvEmformerModel(AAssetManager *mgr,
-                                     const ModelConfig &config)
-    : num_threads_(config.num_threads) {
+                                     const ModelConfig &config) {
   InitEncoder(mgr, config.encoder_param, config.encoder_bin);
   InitDecoder(mgr, config.decoder_param, config.decoder_bin);
   InitJoiner(mgr, config.joiner_param, config.joiner_bin);
@@ -63,7 +61,6 @@ ConvEmformerModel::ConvEmformerModel(AAssetManager *mgr,
 std::pair<ncnn::Mat, std::vector<ncnn::Mat>> ConvEmformerModel::RunEncoder(
     ncnn::Mat &features, const std::vector<ncnn::Mat> &states) {
   ncnn::Extractor encoder_ex = encoder_.create_extractor();
-  encoder_ex.set_num_threads(num_threads_);
   return RunEncoder(features, states, &encoder_ex);
 }
 
@@ -99,7 +96,6 @@ std::pair<ncnn::Mat, std::vector<ncnn::Mat>> ConvEmformerModel::RunEncoder(
 
 ncnn::Mat ConvEmformerModel::RunDecoder(ncnn::Mat &decoder_input) {
   ncnn::Extractor decoder_ex = decoder_.create_extractor();
-  decoder_ex.set_num_threads(num_threads_);
   return RunDecoder(decoder_input, &decoder_ex);
 }
 
@@ -116,7 +112,6 @@ ncnn::Mat ConvEmformerModel::RunDecoder(ncnn::Mat &decoder_input,
 ncnn::Mat ConvEmformerModel::RunJoiner(ncnn::Mat &encoder_out,
                                        ncnn::Mat &decoder_out) {
   auto joiner_ex = joiner_.create_extractor();
-  joiner_ex.set_num_threads(num_threads_);
   return RunJoiner(encoder_out, decoder_out, &joiner_ex);
 }
 

sherpa-ncnn/csrc/conv-emformer-model.h

@@ -85,8 +85,6 @@ class ConvEmformerModel : public Model {
   ncnn::Net decoder_;
   ncnn::Net joiner_;
 
-  int32_t num_threads_;
-
   int32_t num_layers_ = 12;               // arg1
   int32_t memory_size_ = 32;              // arg2
   int32_t cnn_module_kernel_ = 31;        // arg3

sherpa-ncnn/csrc/generate-int8-scale-table.cc

@@ -995,7 +995,6 @@ int main(int argc, char **argv) {
   config.decoder_bin = argv[4];
   config.joiner_param = argv[5];
   config.joiner_bin = argv[6];
-  config.num_threads = num_threads;
 
   const char *encoder_scale_table = argv[7];
   const char *joiner_scale_table = argv[8];

sherpa-ncnn/csrc/lstm-model.cc

@@ -24,8 +24,7 @@
 
 namespace sherpa_ncnn {
 
-LstmModel::LstmModel(const ModelConfig &config)
-    : num_threads_(config.num_threads) {
+LstmModel::LstmModel(const ModelConfig &config) {
   encoder_.opt = config.encoder_opt;
   decoder_.opt = config.decoder_opt;
   joiner_.opt = config.joiner_opt;
@@ -56,8 +55,7 @@ LstmModel::LstmModel(const ModelConfig &config)
 }
 
 #if __ANDROID_API__ >= 9
-LstmModel::LstmModel(AAssetManager *mgr, const ModelConfig &config)
-    : num_threads_(config.num_threads) {
+LstmModel::LstmModel(AAssetManager *mgr, const ModelConfig &config) {
   InitEncoder(mgr, config.encoder_param, config.encoder_bin);
   InitDecoder(mgr, config.decoder_param, config.decoder_bin);
   InitJoiner(mgr, config.joiner_param, config.joiner_bin);
@@ -105,13 +103,11 @@ std::pair<ncnn::Mat, std::vector<ncnn::Mat>> LstmModel::RunEncoder(
 std::pair<ncnn::Mat, std::vector<ncnn::Mat>> LstmModel::RunEncoder(
     ncnn::Mat &features, const std::vector<ncnn::Mat> &states) {
   ncnn::Extractor encoder_ex = encoder_.create_extractor();
-  encoder_ex.set_num_threads(num_threads_);
   return RunEncoder(features, states, &encoder_ex);
 }
 
 ncnn::Mat LstmModel::RunDecoder(ncnn::Mat &decoder_input) {
   ncnn::Extractor decoder_ex = decoder_.create_extractor();
-  decoder_ex.set_num_threads(num_threads_);
   return RunDecoder(decoder_input, &decoder_ex);
 }
 
@@ -127,7 +123,6 @@ ncnn::Mat LstmModel::RunDecoder(ncnn::Mat &decoder_input,
 
 ncnn::Mat LstmModel::RunJoiner(ncnn::Mat &encoder_out, ncnn::Mat &decoder_out) {
   auto joiner_ex = joiner_.create_extractor();
-  joiner_ex.set_num_threads(num_threads_);
   return RunJoiner(encoder_out, decoder_out, &joiner_ex);
 }
 

sherpa-ncnn/csrc/lstm-model.h

@@ -117,8 +117,6 @@ class LstmModel : public Model {
 
   std::vector<int32_t> joiner_input_indexes_;
   std::vector<int32_t> joiner_output_indexes_;
-
-  int32_t num_threads_;
 };
 
 }  // namespace sherpa_ncnn

sherpa-ncnn/csrc/model.cc

@@ -38,7 +38,9 @@ std::string ModelConfig::ToString() const {
 
   os << "tokens: " << tokens << "\n";
 
-  os << "num_threads: " << num_threads << "\n";
+  os << "encoder num_threads: " << encoder_opt.num_threads << "\n";
+  os << "decoder num_threads: " << decoder_opt.num_threads << "\n";
+  os << "joiner num_threads: " << joiner_opt.num_threads << "\n";
 
   return os.str();
 }

sherpa-ncnn/csrc/model.h

@@ -36,7 +36,6 @@ struct ModelConfig {
   std::string joiner_param;   // path to joiner.ncnn.param
   std::string joiner_bin;     // path to joiner.ncnn.bin
   std::string tokens;         // path to tokens.txt
-  int32_t num_threads;        // number of threads to run the model
   bool use_vulkan_compute = false;
 
   ncnn::Option encoder_opt;

sherpa-ncnn/csrc/sherpa-ncnn-microphone.cc

@@ -81,11 +81,15 @@ https://huggingface.co/csukuangfj/sherpa-ncnn-2022-09-05
   config.joiner_param = argv[6];
   config.joiner_bin = argv[7];
 
-  config.num_threads = 4;
+  int32_t num_threads = 4;
   if (argc == 9) {
-    config.num_threads = atoi(argv[8]);
+    num_threads = atoi(argv[8]);
   }
 
+  config.encoder_opt.num_threads = num_threads;
+  config.decoder_opt.num_threads = num_threads;
+  config.joiner_opt.num_threads = num_threads;
+
   sherpa_ncnn::SymbolTable sym(config.tokens);
   fprintf(stderr, "%s\n", config.ToString().c_str());
 

sherpa-ncnn/csrc/sherpa-ncnn.cc

@@ -60,10 +60,13 @@ https://huggingface.co/csukuangfj/sherpa-ncnn-2022-09-05
 
   std::string wav_filename = argv[8];
 
-  config.num_threads = 4;
+  int32_t num_threads = 4;
   if (argc == 10) {
-    config.num_threads = atoi(argv[9]);
+    num_threads = atoi(argv[9]);
   }
+  config.encoder_opt.num_threads = num_threads;
+  config.decoder_opt.num_threads = num_threads;
+  config.joiner_opt.num_threads = num_threads;
 
   float expected_sampling_rate = 16000;
 

sherpa-ncnn/jni/jni.cc

@@ -164,7 +164,11 @@ static ModelConfig GetModelConfig(JNIEnv *env, jobject config) {
   env->ReleaseStringUTFChars(s, p);
 
   fid = env->GetFieldID(cls, "numThreads", "I");
-  model_config.num_threads = env->GetIntField(config, fid);
+
+  int32_t num_threads = env->GetIntField(config, fid);
+  model_config.encoder_opt.num_threads = num_threads;
+  model_config.decoder_opt.num_threads = num_threads;
+  model_config.joiner_opt.num_threads = num_threads;
 
   fid = env->GetFieldID(cls, "useGPU", "Z");
   model_config.use_vulkan_compute = env->GetBooleanField(config, fid);