diff --git a/torchaudio/csrc/CMakeLists.txt b/torchaudio/csrc/CMakeLists.txt
index c6f8edc89d..42f00e31f0 100644
--- a/torchaudio/csrc/CMakeLists.txt
+++ b/torchaudio/csrc/CMakeLists.txt
@@ -1,9 +1,3 @@
-# the following line is added in order to export symbols when building on Windows
-# this approach has some limitations as documented in https://github.com/pytorch/pytorch/pull/3650
-if (MSVC)
-  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
-endif()
-
 ################################################################################
 # libtorchaudio
 ################################################################################
@@ -205,7 +199,7 @@ if(USE_FFMPEG)
     "${LIBTORCHAUDIO_FFMPEG_SOURCES}"
     "${LIBTORCHAUDIO_INCLUDE_DIRS};${FFMPEG_INCLUDE_DIRS}"
     "torch;${FFMPEG_LIBRARIES}"
-    "${LIBTORCHAUDIO_COMPILE_DEFINITIONS}"
+    "${LIBTORCHAUDIO_COMPILE_DEFINITIONS};TORCHAUDIO_BUILD_MAIN_LIB"
   )
 endif()
 
diff --git a/torchaudio/csrc/export.h b/torchaudio/csrc/export.h
new file mode 100644
index 0000000000..b66b571880
--- /dev/null
+++ b/torchaudio/csrc/export.h
@@ -0,0 +1,35 @@
+#pragma once
+
+// Define the visibility of symbols.
+// The original logic and background can be found here.
+// https://github.com/pytorch/pytorch/blob/bcc02769bef1d7b89bec724223284958b7c5b564/c10/macros/Export.h#L49-L55
+//
+// In the context of torchaudio, the logic is simpler at the moment.
+//
+// The torchaudio custom operations are implemented in
+// `torchaudio/lib/libtorchaudio.[so|pyd]`. Some symbols are referred from
+// `torchaudio._torchaudio`.
+//
+// In Windows, default visibility of dynamically library are hidden, while in
+// Linux/macOS, they are visible.
+//
+// At the moment we do not expect torchaudio libraries to be built/linked
+// statically. We assume they are always shared.
+
+#ifdef _WIN32
+#define TORCHAUDIO_EXPORT __declspec(dllexport)
+#define TORCHAUDIO_IMPORT __declspec(dllimport)
+#else // _WIN32
+#if defined(__GNUC__)
+#define TORCHAUDIO_EXPORT __attribute__((__visibility__("default")))
+#else // defined(__GNUC__)
+#define TORCHAUDIO_EXPORT
+#endif // defined(__GNUC__)
+#define TORCHAUDIO_IMPORT TORCHAUDIO_EXPORT
+#endif // _WIN32
+
+#ifdef TORCHAUDIO_BUILD_MAIN_LIB
+#define TORCHAUDIO_API TORCHAUDIO_EXPORT
+#else
+#define TORCHAUDIO_API TORCHAUDIO_IMPORT
+#endif
diff --git a/torchaudio/csrc/ffmpeg/ffmpeg.h b/torchaudio/csrc/ffmpeg/ffmpeg.h
index 6ecab63673..e11e8ab8ce 100644
--- a/torchaudio/csrc/ffmpeg/ffmpeg.h
+++ b/torchaudio/csrc/ffmpeg/ffmpeg.h
@@ -1,6 +1,7 @@
 // One stop header for all ffmepg needs
 #pragma once
 #include <torch/torch.h>
+#include <torchaudio/csrc/export.h>
 #include <cstdint>
 #include <map>
 #include <memory>
@@ -85,30 +86,30 @@ void clean_up_dict(AVDictionary* p);
 // AVFormatContext
 ////////////////////////////////////////////////////////////////////////////////
 struct AVFormatContextDeleter {
-  void operator()(AVFormatContext* p);
+  TORCHAUDIO_API void operator()(AVFormatContext* p);
 };
 
 struct AVFormatContextPtr
     : public Wrapper<AVFormatContext, AVFormatContextDeleter> {
-  explicit AVFormatContextPtr(AVFormatContext* p);
+  TORCHAUDIO_API explicit AVFormatContextPtr(AVFormatContext* p);
 };
 
 ////////////////////////////////////////////////////////////////////////////////
 // AVIO
 ////////////////////////////////////////////////////////////////////////////////
 struct AVIOContextDeleter {
-  void operator()(AVIOContext* p);
+  TORCHAUDIO_API void operator()(AVIOContext* p);
 };
 
 struct AVIOContextPtr : public Wrapper<AVIOContext, AVIOContextDeleter> {
-  explicit AVIOContextPtr(AVIOContext* p);
+  TORCHAUDIO_API explicit AVIOContextPtr(AVIOContext* p);
 };
 
 ////////////////////////////////////////////////////////////////////////////////
 // AVPacket
 ////////////////////////////////////////////////////////////////////////////////
 struct AVPacketDeleter {
-  void operator()(AVPacket* p);
+  TORCHAUDIO_API void operator()(AVPacket* p);
 };
 
 struct AVPacketPtr : public Wrapper<AVPacket, AVPacketDeleter> {
@@ -137,7 +138,7 @@ struct AutoPacketUnref {
 // AVFrame
 ////////////////////////////////////////////////////////////////////////////////
 struct AVFrameDeleter {
-  void operator()(AVFrame* p);
+  TORCHAUDIO_API void operator()(AVFrame* p);
 };
 
 struct AVFramePtr : public Wrapper<AVFrame, AVFrameDeleter> {
@@ -149,7 +150,7 @@ struct AVFramePtr : public Wrapper<AVFrame, AVFrameDeleter> {
 // of AVBufferRefPtr.
 ////////////////////////////////////////////////////////////////////////////////
 struct AutoBufferUnref {
-  void operator()(AVBufferRef* p);
+  TORCHAUDIO_API void operator()(AVBufferRef* p);
 };
 
 struct AVBufferRefPtr : public Wrapper<AVBufferRef, AutoBufferUnref> {
@@ -161,7 +162,7 @@ struct AVBufferRefPtr : public Wrapper<AVBufferRef, AutoBufferUnref> {
 // AVCodecContext
 ////////////////////////////////////////////////////////////////////////////////
 struct AVCodecContextDeleter {
-  void operator()(AVCodecContext* p);
+  TORCHAUDIO_API void operator()(AVCodecContext* p);
 };
 struct AVCodecContextPtr
     : public Wrapper<AVCodecContext, AVCodecContextDeleter> {
@@ -172,7 +173,7 @@ struct AVCodecContextPtr
 // AVFilterGraph
 ////////////////////////////////////////////////////////////////////////////////
 struct AVFilterGraphDeleter {
-  void operator()(AVFilterGraph* p);
+  TORCHAUDIO_API void operator()(AVFilterGraph* p);
 };
 struct AVFilterGraphPtr : public Wrapper<AVFilterGraph, AVFilterGraphDeleter> {
   AVFilterGraphPtr();
diff --git a/torchaudio/csrc/ffmpeg/stream_reader.h b/torchaudio/csrc/ffmpeg/stream_reader.h
index 0cdf68e4eb..285f9ee74d 100644
--- a/torchaudio/csrc/ffmpeg/stream_reader.h
+++ b/torchaudio/csrc/ffmpeg/stream_reader.h
@@ -1,4 +1,5 @@
 #pragma once
+#include <torchaudio/csrc/export.h>
 #include <torchaudio/csrc/ffmpeg/decoder.h>
 #include <torchaudio/csrc/ffmpeg/stream_processor.h>
 #include <torchaudio/csrc/ffmpeg/typedefs.h>
@@ -18,7 +19,7 @@ class StreamReader {
   std::vector<std::pair<int, int>> stream_indices;
 
  public:
-  explicit StreamReader(AVFormatContextPtr&& p);
+  TORCHAUDIO_API explicit StreamReader(AVFormatContextPtr&& p);
   ~StreamReader() = default;
   // Non-copyable
   StreamReader(const StreamReader&) = delete;
@@ -41,32 +42,32 @@ class StreamReader {
   //////////////////////////////////////////////////////////////////////////////
  public:
   // Find a suitable audio/video streams using heuristics from ffmpeg
-  int64_t find_best_audio_stream() const;
-  int64_t find_best_video_stream() const;
+  TORCHAUDIO_API int64_t find_best_audio_stream() const;
+  TORCHAUDIO_API int64_t find_best_video_stream() const;
   // Fetch metadata of the source
-  c10::Dict<std::string, std::string> get_metadata() const;
+  TORCHAUDIO_API c10::Dict<std::string, std::string> get_metadata() const;
   // Fetch information about source streams
-  int64_t num_src_streams() const;
-  SrcStreamInfo get_src_stream_info(int i) const;
+  TORCHAUDIO_API int64_t num_src_streams() const;
+  TORCHAUDIO_API SrcStreamInfo get_src_stream_info(int i) const;
   // Fetch information about output streams
-  int64_t num_out_streams() const;
-  OutputStreamInfo get_out_stream_info(int i) const;
+  TORCHAUDIO_API int64_t num_out_streams() const;
+  TORCHAUDIO_API OutputStreamInfo get_out_stream_info(int i) const;
   // Check if all the buffers of the output streams are ready.
-  bool is_buffer_ready() const;
+  TORCHAUDIO_API bool is_buffer_ready() const;
 
   //////////////////////////////////////////////////////////////////////////////
   // Configure methods
   //////////////////////////////////////////////////////////////////////////////
-  void seek(double timestamp);
+  TORCHAUDIO_API void seek(double timestamp);
 
-  void add_audio_stream(
+  TORCHAUDIO_API void add_audio_stream(
       int64_t i,
       int64_t frames_per_chunk,
       int64_t num_chunks,
       const c10::optional<std::string>& filter_desc,
       const c10::optional<std::string>& decoder,
       const OptionDict& decoder_option);
-  void add_video_stream(
+  TORCHAUDIO_API void add_video_stream(
       int64_t i,
       int64_t frames_per_chunk,
       int64_t num_chunks,
@@ -74,7 +75,7 @@ class StreamReader {
       const c10::optional<std::string>& decoder,
       const OptionDict& decoder_option,
       const c10::optional<std::string>& hw_accel);
-  void remove_stream(int64_t i);
+  TORCHAUDIO_API void remove_stream(int64_t i);
 
  private:
   void add_stream(
@@ -91,15 +92,15 @@ class StreamReader {
   //////////////////////////////////////////////////////////////////////////////
   // Stream methods
   //////////////////////////////////////////////////////////////////////////////
-  int process_packet();
-  int process_packet_block(double timeout, double backoff);
+  TORCHAUDIO_API int process_packet();
+  TORCHAUDIO_API int process_packet_block(double timeout, double backoff);
 
-  int drain();
+  TORCHAUDIO_API int drain();
 
   //////////////////////////////////////////////////////////////////////////////
   // Retrieval
   //////////////////////////////////////////////////////////////////////////////
-  std::vector<c10::optional<torch::Tensor>> pop_chunks();
+  TORCHAUDIO_API std::vector<c10::optional<torch::Tensor>> pop_chunks();
 };
 
 } // namespace ffmpeg
diff --git a/torchaudio/csrc/ffmpeg/stream_reader_wrapper.h b/torchaudio/csrc/ffmpeg/stream_reader_wrapper.h
index fada6417f7..c53b452ce2 100644
--- a/torchaudio/csrc/ffmpeg/stream_reader_wrapper.h
+++ b/torchaudio/csrc/ffmpeg/stream_reader_wrapper.h
@@ -1,12 +1,13 @@
 #pragma once
 #include <torch/script.h>
+#include <torchaudio/csrc/export.h>
 #include <torchaudio/csrc/ffmpeg/stream_reader.h>
 
 namespace torchaudio {
 namespace ffmpeg {
 
 // create format context for reading media
-AVFormatContextPtr get_input_format_context(
+TORCHAUDIO_API AVFormatContextPtr get_input_format_context(
     const std::string& src,
     const c10::optional<std::string>& device,
     const OptionDict& option,
@@ -65,16 +66,16 @@ using OutInfo = std::tuple<
 // suitable for Binding the code (i.e. it receives/returns pritimitves)
 struct StreamReaderBinding : public StreamReader,
                              public torch::CustomClassHolder {
-  explicit StreamReaderBinding(AVFormatContextPtr&& p);
-  SrcInfo get_src_stream_info(int64_t i);
-  SrcInfoPyBind get_src_stream_info_pybind(int64_t i);
-  OutInfo get_out_stream_info(int64_t i);
+  TORCHAUDIO_API explicit StreamReaderBinding(AVFormatContextPtr&& p);
+  TORCHAUDIO_API SrcInfo get_src_stream_info(int64_t i);
+  TORCHAUDIO_API SrcInfoPyBind get_src_stream_info_pybind(int64_t i);
+  TORCHAUDIO_API OutInfo get_out_stream_info(int64_t i);
 
-  int64_t process_packet(
+  TORCHAUDIO_API int64_t process_packet(
       const c10::optional<double>& timeout = c10::optional<double>(),
       const double backoff = 10.);
 
-  void process_all_packets();
+  TORCHAUDIO_API void process_all_packets();
 };
 
 } // namespace ffmpeg