Add OpenSSL support for digest and signatures

Autotools: add --with-crypto=openssl
This enables RPM to locate the appropriate flags for compiling
against OpenSSL for digest and hash functions.

This implementation changes the old behavior of
--with[out]-beecrypt toggling between beecrypt and nss. It will
now throw an error if attempting to use --with-beecrypt
indicating that the user should instead use --with-crypto=

See also:
rpm-software-management#119

INSTALL

@@ -9,17 +9,34 @@ The libmagic (aka file) library for file type detection (used by rpmbuild).
 The source for the file utility + library is available from
     ftp://ftp.astron.com/pub/file/
 
-The NSS >= 3.12 library for encryption, and NSPR library which NSS uses. 
-Both NSPR and NSS libraries and headers need to be installed during RPM
-compilation. As NSPR and NSS typically install their headers outside
-the regular include search path, you need to tell configure about this,
-eg something like:
+You will need a cryptographic library to support digests and signatures.
+This library may be Mozilla NSS, OpenSSL or beecrypt. Which library to use
+must be specified with the --with-crypto=[beecrypt|nss|openssl] argument
+to configure.
+
+If using the Mozilla NSS library for encyption (and NSPR library which
+NSS uses) it must be version 3.12 or later. Both NSPR and NSS libraries and
+headers need to be installed during RPM compilation. As NSPR and NSS
+typically install their headers outside the regular include search path,
+you need to tell configure about this, eg something like:
     ./configure <......> CPPFLAGS="-I/usr/include/nspr -I/usr/include/nss"
 
 The NSPR and NSS libraries are available from 
     http://www.mozilla.org/projects/security/pki/nss/
     http://www.mozilla.org/projects/nspr/
 
+If using the OpenSSL library for encryption, it must be version 1.0.2 or
+later. Note: when compiling against OpenSSL, there is a possible license
+incompatibility. For more details on this, see
+https://people.gnome.org/~markmc/openssl-and-the-gpl.html
+Some Linux distributions have different legal interpretations of this
+possible incompatibility. It is recommended to consult with a lawyer before
+building RPM against OpenSSL.
+Fedora: https://fedoraproject.org/wiki/Licensing:FAQ#What.27s_the_deal_with_the_OpenSSL_license.3F
+Debian: https://lists.debian.org/debian-legal/2002/10/msg00113.html
+
+The OpenSSL crypto library is available from https://www.openssl.org/
+
 The Berkeley DB >= 4.3.x (4.5.x or newer recommended) is required for the
 default database backend. BDB can be downloaded from
 http://www.oracle.com/technology/software/products/berkeley-db/index.html

Makefile.am

@@ -43,6 +43,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/lib
 AM_CPPFLAGS += -I$(top_srcdir)/rpmio
 AM_CPPFLAGS += @WITH_BEECRYPT_INCLUDE@
 AM_CPPFLAGS += @WITH_NSS_INCLUDE@
+AM_CPPFLAGS += @WITH_OPENSSL_INCLUDE@
 AM_CPPFLAGS += @WITH_POPT_INCLUDE@
 AM_CPPFLAGS += -I$(top_srcdir)/misc
 AM_CPPFLAGS += -DLOCALEDIR="\"$(localedir)\""
@@ -111,45 +112,45 @@ rpm_SOURCES =		rpmqv.c debug.h system.h
 rpm_CPPFLAGS =		$(AM_CPPFLAGS) -DIAM_RPMEIU -DIAM_RPMQ -DIAM_RPMV
 rpm_LDADD =		libcliutils.la
 rpm_LDADD +=		lib/librpm.la rpmio/librpmio.la
-rpm_LDADD +=		@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
+rpm_LDADD +=		@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_OPENSSL_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
 
 rpmdb_SOURCES =		rpmdb.c debug.h system.h
 rpmdb_CPPFLAGS =	$(AM_CPPFLAGS)
 rpmdb_LDADD =		libcliutils.la
 rpmdb_LDADD +=		lib/librpm.la rpmio/librpmio.la
-rpmdb_LDADD +=		@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
+rpmdb_LDADD +=		@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_OPENSSL_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
 
 rpmkeys_SOURCES =	rpmkeys.c debug.h system.h
 rpmkeys_CPPFLAGS =	$(AM_CPPFLAGS)
 rpmkeys_LDADD =		libcliutils.la
 rpmkeys_LDADD +=	lib/librpm.la rpmio/librpmio.la
-rpmkeys_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
+rpmkeys_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_OPENSSL_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
 
 rpmsign_SOURCES =	rpmsign.c debug.h system.h
 rpmsign_CPPFLAGS =	$(AM_CPPFLAGS)
 rpmsign_LDADD =		libcliutils.la
 rpmsign_LDADD +=	sign/librpmsign.la lib/librpm.la rpmio/librpmio.la
-rpmsign_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
+rpmsign_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_OPENSSL_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
 
 rpmbuild_SOURCES =	rpmbuild.c debug.h system.h
 rpmbuild_CPPFLAGS =	$(AM_CPPFLAGS)
 rpmbuild_LDADD =	libcliutils.la
 rpmbuild_LDADD +=	build/librpmbuild.la lib/librpm.la rpmio/librpmio.la
-rpmbuild_LDADD += 	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
+rpmbuild_LDADD += 	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_OPENSSL_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
 
 rpmspec_SOURCES =	rpmspec.c debug.h system.h
 rpmspec_CPPFLAGS =	$(AM_CPPFLAGS)
 rpmspec_LDADD =		libcliutils.la
 rpmspec_LDADD +=	build/librpmbuild.la lib/librpm.la rpmio/librpmio.la
-rpmspec_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
+rpmspec_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_OPENSSL_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
 
 rpm2cpio_SOURCES =	rpm2cpio.c debug.h system.h
 rpm2cpio_LDADD =	lib/librpm.la rpmio/librpmio.la
-rpm2cpio_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
+rpm2cpio_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_OPENSSL_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@
 
 rpm2archive_SOURCES =	rpm2archive.c debug.h system.h
 rpm2archive_LDADD =	lib/librpm.la rpmio/librpmio.la
-rpm2archive_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@ @WITH_ARCHIVE_LIB@
+rpm2archive_LDADD +=	@WITH_BEECRYPT_LIB@ @WITH_NSS_LIB@ @WITH_OPENSSL_LIB@ @WITH_POPT_LIB@ @WITH_ZLIB_LIB@ @WITH_ARCHIVE_LIB@
 
 
 if LIBELF

configure.ac

@@ -245,18 +245,30 @@ AC_CHECK_HEADERS([dwarf.h], [
 AM_CONDITIONAL(LIBDWARF,[test "$WITH_LIBDWARF" = yes])
 
 #=================
+# Select crypto library
+AC_ARG_WITH(crypto,
+            [AC_HELP_STRING([--with-crypto=CRYPTO_LIB],
+                            [The cryptographic library to use (nss|beecrypt|openssl). The default is nss.])
+                            ],[],
+                            [with_crypto=nss])
+
+# Refuse to proceed if someone specified --with-beecrypt (removed)
+AC_ARG_WITH(beecrypt,
+    [AC_HELP_STRING([--with-beecrypt (OBSOLETE)], [Obsolete argument. Use --with-crypto=beecrypt])
+    ],[AC_MSG_ERROR([--with-beecrypt no longer supported. Use --with-crypto=beecrypt])],
+    [])
+
 # Check for beecrypt library if requested.
-AC_ARG_WITH(beecrypt, [  --with-beecrypt         build with beecrypt support ],,[with_beecrypt=no])
 AC_ARG_WITH(internal_beecrypt, [  --with-internal-beecrypt build with internal beecrypt library ],,[with_internal_beecrypt=no])
 AM_CONDITIONAL([WITH_INTERNAL_BEECRYPT],[test "$with_internal_beecrypt" = yes])
 if test "$with_internal_beecrypt" = yes ; then
-  with_beecrypt=yes
+  with_crypto=beecrypt
 fi
-AM_CONDITIONAL([WITH_BEECRYPT],[test "$with_beecrypt" = yes])
+AM_CONDITIONAL([WITH_BEECRYPT],[test "$with_crypto" = beecrypt])
 
 WITH_BEECRYPT_INCLUDE=
 WITH_BEECRYPT_LIB=
-if test "$with_beecrypt" = yes ; then
+if test "$with_crypto" = beecrypt ; then
   AC_DEFINE(WITH_BEECRYPT, 1, [Build with beecrypt instead of nss3 support?])
   if test "$with_internal_beecrypt" = yes ; then
     WITH_BEECRYPT_INCLUDE="-I\$(top_srcdir)/beecrypt"
@@ -265,7 +277,7 @@ if test "$with_beecrypt" = yes ; then
     AC_CHECK_LIB(beecrypt, mpfprintln, [
       WITH_BEECRYPT_LIB="-lbeecrypt"
     ],[
-      AC_MSG_ERROR([missing required library 'beecrypt']) 
+      AC_MSG_ERROR([missing required library 'beecrypt'])
     ])
     AC_CHECK_HEADER([beecrypt/api.h], [AC_DEFINE(HAVE_BEECRYPT_API_H, 1, [Define to 1 if you have the <beecrypt/api.h> header file.])
     ])
@@ -274,14 +286,94 @@ fi
 AC_SUBST(WITH_BEECRYPT_LIB)
 AC_SUBST(WITH_BEECRYPT_INCLUDE)
 
+#=================
+# Check for OpenSSL library.
+# We need evp.h from OpenSSL.
+
+WITH_OPENSSL_INCLUDE=
+WITH_OPENSSL_LIB=
+if test "$with_crypto" = openssl; then
+# If we have pkgconfig make sure CPPFLAGS are setup correctly for the OpenSSL
+# -I include path.
+AC_PATH_TOOL([PKGCONFIG], [pkg-config], [no], [$PATH:/usr/bin:/usr/local/bin])
+if test "x$PKGCONFIG" != "xno"; then
+  CPPFLAGS="$CPPFLAGS $($PKGCONFIG --cflags libcrypto)"
+  WITH_OPENSSL_LIB=$($PKGCONFIG --libs libcrypto)
+else
+  WITH_OPENSSL_LIB=-lcrypto
+fi
+
+AC_CHECK_HEADERS([openssl/evp.h], [], [
+  AC_MSG_ERROR([missing required OpenSSL header])
+])
+AC_CHECK_HEADERS([openssl/rsa.h], [], [
+  AC_MSG_ERROR([missing required OpenSSL header])
+])
+AC_CHECK_HEADERS([openssl/dsa.h], [], [
+  AC_MSG_ERROR([missing required OpenSSL header])
+])
+
+AC_CHECK_LIB(crypto, EVP_DigestInit_ex, [], [
+  AC_MSG_ERROR([required OpenSSL library 'libcrypto' missing or too old])
+])
+
+AC_CHECK_LIB(crypto, EVP_MD_CTX_new, [
+    AC_DEFINE(HAVE_EVP_MD_CTX_NEW, 1, [Define to 1 if OpenSSL has EVP_MD_CTX_new])
+    AC_SUBST(HAVE_EVP_MD_CTX_NEW, [1])
+  ], [
+  AC_CHECK_LIB(crypt, EVP_MD_CTX_create, [], [
+      AC_MSG_ERROR([required OpenSSL library 'libcrypto' missing or too old])
+  ])
+])
+
+AC_CHECK_LIB(crypto, EVP_PKEY_CTX_new, [], [
+  AC_MSG_ERROR([required OpenSSL library 'libcrypto' missing or too old])
+])
+
+AC_CHECK_LIB(crypto, DSA_set0_key, [
+    AC_DEFINE(HAVE_DSA_SET0_KEY, 1, [Define to 1 if OpenSSL has DSA_set0_key])
+    AC_SUBST(HAVE_DSA_SET0_KEY, [1])
+    ], []
+)
+
+AC_CHECK_LIB(crypto, DSA_set0_pqg, [
+    AC_DEFINE(HAVE_DSA_SET0_PQG, 1, [Define to 1 if OpenSSL has DSA_set0_pqg])
+    AC_SUBST(HAVE_DSA_SET0_PQG, [1])
+    ], []
+)
+
+AC_CHECK_LIB(crypto, DSA_SIG_set0, [
+    AC_DEFINE(HAVE_DSA_SIG_SET0, 1, [Define to 1 if OpenSSL has DSA_SIG_set0])
+    AC_SUBST(HAVE_DSA_SIG_SET0, [1])
+    ], []
+)
+
+AC_CHECK_LIB(crypto, RSA_set0_key, [
+    AC_DEFINE(HAVE_RSA_SET0_KEY, 1, [Define to 1 if OpenSSL has RSA_set0_key])
+    AC_SUBST(HAVE_RSA_SET0_KEY, [1])
+    ], []
+)
+
+AC_CHECK_LIB(crypto, BN_bn2binpad, [
+    AC_DEFINE(HAVE_BN2BINPAD, 1, [Define to 1 if OpenSSL has BN_bn2binpad])
+    AC_SUBST(HAVE_BN2BINPAD, [1])
+    ], []
+)
+
+fi
+
+AM_CONDITIONAL([WITH_OPENSSL],[test "$with_crypto" = openssl])
+AC_SUBST(WITH_OPENSSL_INCLUDE)
+AC_SUBST(WITH_OPENSSL_LIB)
+
 #=================
 # Check for NSS library.
-# We need nss.h from NSS which needs nspr.h. Unfortunately both glibc and NSS 
-# have a header named nss.h... so make extra check for NSS's sechash.h 
+# We need nss.h from NSS which needs nspr.h. Unfortunately both glibc and NSS
+# have a header named nss.h... so make extra check for NSS's sechash.h
 # which we use too and hopefully is slightly more unique to NSS.
 WITH_NSS_INCLUDE=
 WITH_NSS_LIB=
-if test "$with_beecrypt" != yes ; then
+if test "$with_crypto" = nss; then
 # If we have pkgconfig make sure CPPFLAGS are setup correctly for the nss
 # -I include path. Otherwise the below checks will fail because nspr.h
 # cannot be found.

rpmio/Makefile.am

@@ -6,6 +6,7 @@ AM_CFLAGS = @RPMCFLAGS@
 AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir) -I$(top_builddir)/include/
 AM_CPPFLAGS += @WITH_NSS_INCLUDE@
 AM_CPPFLAGS += @WITH_BEECRYPT_INCLUDE@
+AM_CPPFLAGS += @WITH_OPENSSL_INCLUDE@
 AM_CPPFLAGS += @WITH_POPT_INCLUDE@
 AM_CPPFLAGS += -I$(top_srcdir)/misc
 AM_CPPFLAGS += -DRPMCONFIGDIR="\"@RPMCONFIGDIR@\""
@@ -24,15 +25,20 @@ librpmio_la_SOURCES = \
 if WITH_BEECRYPT
 librpmio_la_SOURCES += digest_beecrypt.c
 else
+if WITH_OPENSSL
+librpmio_la_SOURCES += digest_openssl.c
+else
 librpmio_la_SOURCES += digest_nss.c
 endif
+endif
 
 
 librpmio_la_LDFLAGS = -version-info $(rpm_version_info)
 librpmio_la_LIBADD = \
 	../misc/libmisc.la \
 	@WITH_NSS_LIB@ \
 	@WITH_BEECRYPT_LIB@ \
+	@WITH_OPENSSL_LIB@ \
 	@WITH_BZ2_LIB@ \
 	@WITH_ZLIB_LIB@ \
 	@WITH_LIBELF_LIB@ \