diff --git a/Makefile.am b/Makefile.am
index accf0b94186..7fa4ceeb48d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,7 +1,12 @@
 ACLOCAL_AMFLAGS = -I m4
 INCLUDES = $(SECP_INCLUDES)
 lib_LTLIBRARIES = libsecp256k1.la
-noinst_LTLIBRARIES = libsecp256k1_common.la
+if USE_ASM
+COMMON_LIB = libsecp256k1_common.la
+else
+COMMON_LIB =
+endif
+noinst_LTLIBRARIES = $(COMMON_LIB)
 include_HEADERS = include/secp256k1.h
 noinst_HEADERS =
 noinst_HEADERS += src/scalar.h
@@ -37,14 +42,14 @@ noinst_HEADERS += src/field_impl.h
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = libsecp256k1.pc
 
-libsecp256k1_common_la_SOURCES =
 if USE_ASM
-libsecp256k1_common_la_SOURCES += src/field_5x52_asm.asm
+libsecp256k1_common_la_SOURCES = src/field_5x52_asm.asm
 endif
 
 libsecp256k1_la_SOURCES = src/secp256k1.c
 libsecp256k1_la_CPPFLAGS = -I$(top_srcdir)/include $(SECP_INCLUDES)
-libsecp256k1_la_LIBADD = libsecp256k1_common.la $(SECP_LIBS)
+libsecp256k1_la_LIBADD = $(COMMON_LIB) $(SECP_LIBS)
+
 
 noinst_PROGRAMS =
 if USE_BENCHMARK
@@ -56,7 +61,7 @@ bench_sign_SOURCES = src/bench_sign.c
 bench_sign_LDADD = libsecp256k1.la $(SECP_LIBS)
 bench_sign_LDFLAGS = -static
 bench_inv_SOURCES = src/bench_inv.c
-bench_inv_LDADD = libsecp256k1_common.la $(SECP_LIBS)
+bench_inv_LDADD = $(COMMON_LIB) $(SECP_LIBS)
 bench_inv_LDFLAGS = -static
 endif
 
@@ -64,7 +69,7 @@ if USE_TESTS
 noinst_PROGRAMS += tests
 tests_SOURCES = src/tests.c
 tests_CPPFLAGS = -DVERIFY $(SECP_TEST_INCLUDES)
-tests_LDADD = libsecp256k1_common.la $(SECP_LIBS) $(SECP_TEST_LIBS)
+tests_LDADD = $(COMMON_LIB) $(SECP_LIBS) $(SECP_TEST_LIBS)
 tests_LDFLAGS = -static
 TESTS = tests
 endif