diff --git a/source/configure.in b/source/configure.in
index bd7a31f25d7..19355d41580 100644
--- a/source/configure.in
+++ b/source/configure.in
@@ -1605,9 +1605,8 @@ for i in $LOOK_DIRS ; do
     save_CPPFLAGS=$CPPFLAGS
     CPPFLAGS="$CPPFLAGS -I$i/include"
 dnl This is here to handle -withval stuff for --with-libiconv
-    if test x"$ICONV_PATH_SPEC" = "xyes" ; then
-       LDFLAGS="-L$i/lib"
-    fi
+dnl Perhaps we should always add a -L
+    LDFLAGS="-L$i/lib"
     LIBS=
     export LDFLAGS LIBS CPPFLAGS
 dnl Try to find iconv(3)
@@ -1616,7 +1615,6 @@ dnl Try to find iconv(3)
     if test "$ICONV_FOUND" = yes; then
         LIB_ADD_DIR(LDFLAGS, "$i/lib")
         CFLAGS_ADD_DIR(CPPFLAGS, "$i/include")
-        LIBS="$save_LIBS"
         ICONV_LOCATION=$i
         export LDFLAGS LIBS CPPFLAGS
 dnl Now, check for a working iconv ... we want to do it here because