Pack all sources into a single tar for embedded systems (#96)

* Pack all sources into a single tar for embedded systems

* Update to v1.4.1

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 project(sherpa-ncnn)
 
-set(SHERPA_NCNN_VERSION "1.4.0")
+set(SHERPA_NCNN_VERSION "1.4.1")
 
 # Disable warning about
 #
@@ -36,6 +36,7 @@ option(SHERPA_NCNN_ENABLE_JNI "Whether to build JNI internface" OFF)
 option(SHERPA_NCNN_ENABLE_BINARY "Whether to build the binary sherpa-ncnn" ON)
 option(SHERPA_NCNN_ENABLE_TEST "Whether to build tests" OFF)
 option(SHERPA_NCNN_ENABLE_C_API "Whether to build C API" ON)
+option(SHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE "Whether to generate-int8-scale-table" ON)
 
 if(DEFINED ANDROID_ABI)
   message(STATUS "Set SHERPA_NCNN_ENABLE_JNI to ON for Android")
@@ -49,6 +50,7 @@ message(STATUS "SHERPA_NCNN_ENABLE_JNI ${SHERPA_NCNN_ENABLE_JNI}")
 message(STATUS "SHERPA_NCNN_ENABLE_BINARY ${SHERPA_NCNN_ENABLE_BINARY}")
 message(STATUS "SHERPA_NCNN_ENABLE_TEST ${SHERPA_NCNN_ENABLE_TEST}")
 message(STATUS "SHERPA_NCNN_ENABLE_C_API ${SHERPA_NCNN_ENABLE_C_API}")
+message(STATUS "SHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE ${SHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE}")
 
 if(NOT CMAKE_BUILD_TYPE)
   message(STATUS "No CMAKE_BUILD_TYPE given, default to Release")

build-android-arm64-v8a-with-vulkan.sh

@@ -113,6 +113,9 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake"
     -DNCNN_SYSTEM_GLSLANG=ON \
     -DSHERPA_NCNN_ENABLE_PORTAUDIO=OFF \
     -DSHERPA_NCNN_ENABLE_BINARY=OFF \
+    -DSHERPA_NCNN_ENABLE_TEST=OFF \
+    -DSHERPA_NCNN_ENABLE_C_API=OFF \
+    -DSHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE=OFF \
     -DCMAKE_INSTALL_PREFIX=./install \
     -DANDROID_ABI="arm64-v8a" \
     -DNCNN_VULKAN=ON \

build-android-arm64-v8a.sh

@@ -49,6 +49,9 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake"
     -DBUILD_SHARED_LIBS=ON \
     -DSHERPA_NCNN_ENABLE_PORTAUDIO=OFF \
     -DSHERPA_NCNN_ENABLE_BINARY=OFF \
+    -DSHERPA_NCNN_ENABLE_TEST=OFF \
+    -DSHERPA_NCNN_ENABLE_C_API=OFF \
+    -DSHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE=OFF \
     -DCMAKE_INSTALL_PREFIX=./install \
     -DANDROID_ABI="arm64-v8a" \
     -DANDROID_PLATFORM=android-21 ..

build-android-armv7-eabi.sh

@@ -48,6 +48,9 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake"
     -DBUILD_SHARED_LIBS=ON \
     -DSHERPA_NCNN_ENABLE_PORTAUDIO=OFF \
     -DSHERPA_NCNN_ENABLE_BINARY=OFF \
+    -DSHERPA_NCNN_ENABLE_TEST=OFF \
+    -DSHERPA_NCNN_ENABLE_C_API=OFF \
+    -DSHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE=OFF \
     -DCMAKE_INSTALL_PREFIX=./install \
     -DANDROID_ABI="armeabi-v7a" -DANDROID_ARM_NEON=ON \
     -DANDROID_PLATFORM=android-21 ..

build-android-x86-64.sh

@@ -50,6 +50,9 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake"
     -DBUILD_SHARED_LIBS=ON \
     -DSHERPA_NCNN_ENABLE_PORTAUDIO=OFF \
     -DSHERPA_NCNN_ENABLE_BINARY=OFF \
+    -DSHERPA_NCNN_ENABLE_TEST=OFF \
+    -DSHERPA_NCNN_ENABLE_C_API=OFF \
+    -DSHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE=OFF \
     -DCMAKE_INSTALL_PREFIX=./install \
     -DANDROID_ABI="x86_64" \
     -DANDROID_PLATFORM=android-21 ..

build-m3axpi.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# This script is exclusively for building sherpa-ncnn
+# on MAIX-III AXera-Pi
+# (https://wiki.sipeed.com/hardware/en/maixIII/ax-pi/axpi.html)
+#
+# It is not for cross-compiling.
+#
+# Before running this script, please make sure you have the following
+# files:
+#
+# root@AXERA:~/asr# pwd
+# /root/asr
+#
+# root@AXERA:~/asr# ls -lh
+# total 25M
+# -rw-r--r--  1  501 staff  59K Feb  2 17:01 kaldi-native-fbank-1.11.tar.gz
+# -rw-r--r--  1  501 staff  12M Feb  2 17:01 ncnn-sherpa-0.8.tar.gz
+# drwxr-xr-x 15  501 staff 4.0K Feb  2 21:04 sherpa-ncnn-1.4.0
+#
+#
+# Note: It is OK if the versions of the above files are different.
+# The two `.tar.gz` files must be placed in $HOME/asr
+
+mkdir -p build
+cd build
+
+cmake \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX=./install \
+  -DBUILD_SHARED_LIBS=OFF \
+  -DSHERPA_NCNN_ENABLE_PYTHON=OFF \
+  -DSHERPA_NCNN_ENABLE_PORTAUDIO=OFF \
+  -DSHERPA_NCNN_ENABLE_JNI=OFF \
+  -DSHERPA_NCNN_ENABLE_BINARY=ON \
+  -DSHERPA_NCNN_ENABLE_TEST=OFF \
+  -DSHERPA_NCNN_ENABLE_C_API=OFF \
+  -DSHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE=OFF \
+  ..
+
+make -j 4
+cd ..
+
+echo
+echo "ls -lh ./build/bin"
+# You will find two statically-linked executables in ./build/bin
+ls -lh ./build/bin
+
+echo
+echo "Please refer to"
+echo "https://k2-fsa.github.io/sherpa/ncnn/pretrained_models/index.html"
+echo "to download pre-trained models"

cmake/kaldi-native-fbank.cmake

@@ -1,6 +1,7 @@
 function(download_kaldi_native_fbank)
   include(FetchContent)
 
+  # Please also change ../pack-for-embedded-systems.sh
   set(kaldi_native_fbank_URL  "https://github.com/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.11.tar.gz")
   set(kaldi_native_fbank_HASH "SHA256=e69ae25ef6f30566ef31ca949dd1b0b8ec3a827caeba93a61d82bb848dac5d69")
 
@@ -12,6 +13,8 @@ function(download_kaldi_native_fbank)
     set(kaldi_native_fbank_URL  "file:///Users/fangjun/Downloads/kaldi-native-fbank-1.11.tar.gz")
   elseif(EXISTS "/tmp/kaldi-native-fbank-1.11.tar.gz")
     set(kaldi_native_fbank_URL  "file:///tmp/kaldi-native-fbank-1.11.tar.gz")
+  elseif(EXISTS "$ENV{HOME}/asr/kaldi-native-fbank-1.11.tar.gz")
+    set(kaldi_native_fbank_URL  "file://$ENV{HOME}/asr/kaldi-native-fbank-1.11.tar.gz")
   endif()
 
   set(KALDI_NATIVE_FBANK_BUILD_TESTS OFF CACHE BOOL "" FORCE)

cmake/ncnn.cmake

@@ -5,6 +5,7 @@ function(download_ncnn)
   # The changed code is in
   # https://github.com/csukuangfj/ncnn/pull/7
 
+  # Please also change ../pack-for-embedded-systems.sh
   set(ncnn_URL "https://github.com/csukuangfj/ncnn/archive/refs/tags/sherpa-0.8.tar.gz")
   set(ncnn_HASH "SHA256=f605c48986406800615d00cf14b955e95f73286eadacedb6c3371542540e1df0")
 
@@ -16,6 +17,8 @@ function(download_ncnn)
     set(ncnn_URL  "file:///Users/fangjun/Downloads/ncnn-sherpa-0.8.tar.gz")
   elseif(EXISTS "/tmp/ncnn-sherpa-0.8.tar.gz")
     set(ncnn_URL  "file:///tmp/ncnn-sherpa-0.8.tar.gz")
+  elseif(EXISTS "$ENV{HOME}/asr/ncnn-sherpa-0.8.tar.gz")
+    set(ncnn_URL  "file://$ENV{HOME}/asr/ncnn-sherpa-0.8.tar.gz")
   endif()
 
   FetchContent_Declare(ncnn

cmake/portaudio.cmake

@@ -12,6 +12,8 @@ function(download_portaudio)
     set(portaudio_URL  "file:///Users/fangjun/Downloads/pa_stable_v190700_20210406.tgz")
   elseif(EXISTS "/tmp/pa_stable_v190700_20210406.tgz")
     set(portaudio_URL  "file:///tmp/pa_stable_v190700_20210406.tgz")
+  elseif(EXISTS "$ENV{HOME}/asr/pa_stable_v190700_20210406.tgz")
+    set(portaudio_URL  "file://$ENV{HOME}/asr/pa_stable_v190700_20210406.tgz")
   endif()
 
   if(BUILD_SHARED_LIBS)

cmake/pybind11.cmake

@@ -12,6 +12,8 @@ function(download_pybind11)
     set(pybind11_URL  "file:///Users/fangjun/Downloads/pybind11-2.10.2.tar.gz")
   elseif(EXISTS "/tmp/pybind11-2.10.2.tar.gz")
     set(pybind11_URL  "file:///tmp/pybind11-2.10.2.tar.gz")
+  elseif(EXISTS "$ENV{HOME}/asr/pybind11-2.10.2.tar.gz")
+    set(pybind11_URL  "file://$ENV{HOME}/asr/pybind11-2.10.2.tar.gz")
   endif()
 
   FetchContent_Declare(pybind11

pack-for-embedded-systems.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# This script pack all source code into a single zip file
+# so that you can build it locally without accessing Internet
+# on your board.
+
+set -e
+
+SHERPA_NCNN_VERSION=$(grep "SHERPA_NCNN_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+echo "SHERPA_NCNN_VERSION: ${SHERPA_NCNN_VERSION}"
+
+dir=sherpa-ncnn-all-in-one-for-embedded-systems-${SHERPA_NCNN_VERSION}
+
+# rm -rf $dir
+mkdir -p $dir
+
+pushd $dir
+
+wget \
+  -O sherpa-ncnn-${SHERPA_NCNN_VERSION}.tar.gz \
+  https://github.com/k2-fsa/sherpa-ncnn/archive/refs/tags/v${SHERPA_NCNN_VERSION}.tar.gz
+
+tar xf sherpa-ncnn-${SHERPA_NCNN_VERSION}.tar.gz
+rm -v sherpa-ncnn-${SHERPA_NCNN_VERSION}.tar.gz
+
+# Please also change ./build-m3axpi.sh
+wget \
+  -O kaldi-native-fbank-1.11.tar.gz \
+  https://github.com/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.11.tar.gz
+
+wget \
+  -O ncnn-sherpa-0.8.tar.gz \
+  https://github.com/csukuangfj/ncnn/archive/refs/tags/sherpa-0.8.tar.gz
+
+cat >README.md <<EOF
+Please put files from this folder to the directory \$HOME/asr/
+
+rm -rf \$HOME/asr
+mkdir -p \$HOME/asr
+
+tar xvf sherpa-ncnn-all-in-one-for-embedded-systems-${SHERPA_NCNN_VERSION}.tar.bz2 --strip-components 1 -C \$HOME/asr
+rm sherpa-ncnn-all-in-one-for-embedded-systems-${SHERPA_NCNN_VERSION}.tar.bz2  # to save space
+
+ls -lh \$HOME/asr
+
+It should print something like below:
+
+ls -lh \$HOME/asr
+total 24368
+-rw-r--r--   1 fangjun  staff    59K Feb  2 17:01 kaldi-native-fbank-1.11.tar.gz
+-rw-r--r--   1 fangjun  staff    12M Feb  2 17:01 sherpa-0.8.tar.gz
+drwxr-xr-x  29 fangjun  staff   928B Feb  2 16:05 sherpa-ncnn-${SHERPA_NCNN_VERSION}
+
+# Note: It is OK if the versions of the above files are different.
+# The two .tar.gz files must be placed in \$HOME/asr
+EOF
+
+popd
+
+tar cjf ${dir}.tar.bz2 $dir

sherpa-ncnn/csrc/CMakeLists.txt

@@ -66,8 +66,10 @@ if(NOT SHERPA_NCNN_ENABLE_PYTHON)
       DESTINATION include/sherpa-ncnn/csrc
     )
 
-    add_executable(generate-int8-scale-table generate-int8-scale-table.cc)
-    target_link_libraries(generate-int8-scale-table sherpa-ncnn-core)
+    if(SHERPA_NCNN_ENABLE_GENERATE_INT8_SCALE_TABLE)
+      add_executable(generate-int8-scale-table generate-int8-scale-table.cc)
+      target_link_libraries(generate-int8-scale-table sherpa-ncnn-core)
+    endif()
   endif()
 endif()