Clean up libsox source and build directory (#1161)

mthrok · web-flow · commit 7a36c5573d4f · 2021-01-08T20:46:59.000-05:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -101,14 +101,14 @@ jobs:
 
       - run:
           command: |
-              mkdir -p third_party/archives/
-              wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
+              mkdir -p third_party/sox/archives/
+              wget --no-clobber --directory-prefix=third_party/sox/archives/ $(awk '/URL /{print $2}' third_party/sox/CMakeLists.txt)
       - save_cache:
 
           key: tp-nix-v2-{{ checksum ".cachekey" }}
 
           paths:
-            - third_party/archives
+            - third_party/sox/archives
       - persist_to_workspace:
           root: third_party
           paths:
@@ -411,9 +411,7 @@ jobs:
           paths:
             - conda
             - env
-            - third_party/build
             - third_party/install
-            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -458,9 +456,7 @@ jobs:
           paths:
             - conda
             - env
-            - third_party/build
             - third_party/install
-            - third_party/src
       - run:
           name: Install torchaudio
           command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -573,9 +569,7 @@ jobs:
           paths:
             - conda
             - env
-            - third_party/build
             - third_party/install
-            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -612,9 +606,7 @@ jobs:
           paths:
             - conda
             - env
-            - third_party/build
             - third_party/install
-            - third_party/src
       - run:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh
diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
@@ -101,14 +101,14 @@ jobs:
           {% endraw %}
       - run:
           command: |
-              mkdir -p third_party/archives/
-              wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
+              mkdir -p third_party/sox/archives/
+              wget --no-clobber --directory-prefix=third_party/sox/archives/ $(awk '/URL /{print $2}' third_party/sox/CMakeLists.txt)
       - save_cache:
           {% raw %}
           key: tp-nix-v2-{{ checksum ".cachekey" }}
           {% endraw %}
           paths:
-            - third_party/archives
+            - third_party/sox/archives
       - persist_to_workspace:
           root: third_party
           paths:
@@ -411,9 +411,7 @@ jobs:
           paths:
             - conda
             - env
-            - third_party/build
             - third_party/install
-            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -458,9 +456,7 @@ jobs:
           paths:
             - conda
             - env
-            - third_party/build
             - third_party/install
-            - third_party/src
       - run:
           name: Install torchaudio
           command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
@@ -573,9 +569,7 @@ jobs:
           paths:
             - conda
             - env
-            - third_party/build
             - third_party/install
-            - third_party/src
       - run:
           name: Install torchaudio
           command: .circleci/unittest/linux/scripts/install.sh
@@ -612,9 +606,7 @@ jobs:
           paths:
             - conda
             - env
-            - third_party/build
             - third_party/install
-            - third_party/src
       - run:
           name: Run style check
           command: .circleci/unittest/linux/scripts/run_style_checks.sh
diff --git a/.gitignore b/.gitignore
@@ -123,7 +123,5 @@ examples/interactive_asr/data/*.model
 examples/interactive_asr/data/*.pt
 
 # third parties
-third_party/archives/
 third_party/install/
-third_party/src/
-third_party/tmp/
+third_party/sox/archives/
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
@@ -8,94 +8,9 @@ option(BUILD_TRANSDUCER "Build transducer statically")
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 if (BUILD_SOX)
-include(ExternalProject)
-set(INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
-set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/archives)
-set(COMMON_ARGS --quiet --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc)
-
-ExternalProject_Add(libmad
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz
-  URL_HASH SHA256=bbfac3ed6bfbc2823d3775ebb931087371e142bb0e9bb1bee51a76a6e0078690
-  PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/patch/libmad.patch
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmad/configure ${COMMON_ARGS}
-)
-
-ExternalProject_Add(amr
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://sourceforge.net/projects/opencore-amr/files/opencore-amr/opencore-amr-0.1.5.tar.gz
-  URL_HASH SHA256=2c006cb9d5f651bfb5e60156dbff6af3c9d35c7bbcc9015308c0aff1e14cd341
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/amr/configure ${COMMON_ARGS}
-)
-
-ExternalProject_Add(libmp3lame
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
-  URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libmp3lame/configure ${COMMON_ARGS} --enable-nasm
-)
-
-ExternalProject_Add(libogg
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
-  URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/libogg/configure ${COMMON_ARGS}
-)
-
-ExternalProject_Add(libflac
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libogg
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
-  URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libflac/configure ${COMMON_ARGS} --with-ogg
-)
-
-ExternalProject_Add(libvorbis
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libogg
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
-  URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libvorbis/configure ${COMMON_ARGS} --with-ogg
-)
-
-ExternalProject_Add(libopus
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libogg
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
-  URL_HASH SHA256=65b58e1e25b2a114157014736a3d9dfeaad8d41be1c8179866f144a2fb44ff9d
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libopus/configure ${COMMON_ARGS} --with-ogg
-)
-
-ExternalProject_Add(opusfile
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libopus
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  STAMP_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile-stamp
-  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile
-  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
-  URL_HASH SHA256=118d8601c12dd6a44f52423e68ca9083cc9f2bfe72da7a8c1acb22a80ae3550b
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/opusfile/configure ${COMMON_ARGS} --disable-http
-)
-
-ExternalProject_Add(libsox
-  PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
-  DEPENDS libogg libflac libvorbis opusfile libmp3lame libmad amr
-  DOWNLOAD_DIR ${ARCHIVE_DIR}
-  URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
-  URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c
-  # OpenMP is by default compiled against GNU OpenMP, which conflicts with the version of OpenMP that PyTorch uses.
-  # See https://github.com/pytorch/audio/pull/1026
-  CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_codec_helper.sh ${CMAKE_CURRENT_SOURCE_DIR}/src/libsox/configure ${COMMON_ARGS} --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile --with-opus --with-amrwb --with-amrnb --disable-openmp --without-sndio --without-pulseaudio
-)
+  add_subdirectory(sox)
 endif()
 
 if(BUILD_TRANSDUCER)
-add_subdirectory(transducer)
+  add_subdirectory(transducer)
 endif()
diff --git a/third_party/build_codec_helper.sh b/third_party/build_codec_helper.sh
diff --git a/third_party/sox/CMakeLists.txt b/third_party/sox/CMakeLists.txt
@@ -0,0 +1,95 @@
+include(ExternalProject)
+
+set(INSTALL_DIR ${PROJECT_SOURCE_DIR}/install)
+set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/archives)
+set(COMMON_ARGS --quiet --disable-shared --enable-static --prefix=${INSTALL_DIR} --with-pic --disable-dependency-tracking --disable-debug --disable-examples --disable-doc)
+
+# To pass custom environment variables to ExternalProject_Add command,
+# we need to do `${CMAKE_COMMAND} -E env ${envs} <COMMANAD>`.
+# https://stackoverflow.com/a/62437353
+# We constrcut the custom environment variables here
+set(envs
+  "PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig"
+  "LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}"
+  "CPPFLAGS=-I${INSTALL_DIR}/include $ENV{CPPFLAGS}"
+)
+
+ExternalProject_Add(mad
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz
+  URL_HASH SHA256=bbfac3ed6bfbc2823d3775ebb931087371e142bb0e9bb1bee51a76a6e0078690
+  PATCH_COMMAND patch < ${CMAKE_CURRENT_SOURCE_DIR}/patch/libmad.patch
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/mad/configure ${COMMON_ARGS}
+)
+
+ExternalProject_Add(amr
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://sourceforge.net/projects/opencore-amr/files/opencore-amr/opencore-amr-0.1.5.tar.gz
+  URL_HASH SHA256=2c006cb9d5f651bfb5e60156dbff6af3c9d35c7bbcc9015308c0aff1e14cd341
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/amr/configure ${COMMON_ARGS}
+)
+
+ExternalProject_Add(lame
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz
+  URL_HASH SHA256=24346b4158e4af3bd9f2e194bb23eb473c75fb7377011523353196b19b9a23ff
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/lame/configure ${COMMON_ARGS} --enable-nasm
+)
+
+ExternalProject_Add(ogg
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz
+  URL_HASH SHA256=c2e8a485110b97550f453226ec644ebac6cb29d1caef2902c007edab4308d985
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/ogg/configure ${COMMON_ARGS}
+)
+
+ExternalProject_Add(flac
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ogg
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://ftp.osuosl.org/pub/xiph/releases/flac/flac-1.3.2.tar.xz
+  URL_HASH SHA256=91cfc3ed61dc40f47f050a109b08610667d73477af6ef36dcad31c31a4a8d53f
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/flac/configure ${COMMON_ARGS} --with-ogg --disable-cpplibs
+)
+
+ExternalProject_Add(vorbis
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ogg
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz
+  URL_HASH SHA256=6ed40e0241089a42c48604dc00e362beee00036af2d8b3f46338031c9e0351cb
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/vorbis/configure ${COMMON_ARGS} --with-ogg
+)
+
+ExternalProject_Add(opus
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ogg
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opus-1.3.1.tar.gz
+  URL_HASH SHA256=65b58e1e25b2a114157014736a3d9dfeaad8d41be1c8179866f144a2fb44ff9d
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/opus/configure ${COMMON_ARGS} --with-ogg
+)
+
+ExternalProject_Add(opusfile
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS opus
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://ftp.osuosl.org/pub/xiph/releases/opus/opusfile-0.12.tar.gz
+  URL_HASH SHA256=118d8601c12dd6a44f52423e68ca9083cc9f2bfe72da7a8c1acb22a80ae3550b
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/opusfile/configure ${COMMON_ARGS} --disable-http
+)
+
+ExternalProject_Add(sox
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ogg flac vorbis opusfile lame mad amr
+  DOWNLOAD_DIR ${ARCHIVE_DIR}
+  URL https://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2
+  URL_HASH SHA256=81a6956d4330e75b5827316e44ae381e6f1e8928003c6aa45896da9041ea149c
+  # OpenMP is by default compiled against GNU OpenMP, which conflicts with the version of OpenMP that PyTorch uses.
+  # See https://github.com/pytorch/audio/pull/1026
+  CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/sox/configure ${COMMON_ARGS} --with-lame --with-flac --with-mad --with-oggvorbis --without-alsa --without-coreaudio --without-png --without-oss --without-sndfile --with-opus --with-amrwb --with-amrnb --disable-openmp --without-sndio --without-pulseaudio
+)
diff --git a/third_party/sox/patch/libmad.patch b/third_party/sox/patch/libmad.patch
