libkern: add ilog2 macro

The kernel source contains several definitions of an ilog2 function;
some are slower than necessary, and one of them is incorrect.
Elimininate them all and define an ilog2 macro in libkern to replace
them, in a way that is fast, correct for all argument types, and, in a
GENERIC kernel, includes a check for an invalid zero parameter.

Folks at Microsoft have verified that having a correct ilog2
definition for their MANA driver doesn't break it.

Reviewed by:	alc, markj, mhorne (older version), jhibbits (older version)
Differential Revision:	https://reviews.freebsd.org/D45170
Differential Revision:	https://reviews.freebsd.org/D45235

(cherry picked from commit b0056b31e90029553894d17c441cbb2c06d31412)

sys/amd64/include/cpufunc.h

@@ -65,10 +65,6 @@ breakpoint(void)
 
 #define	bsfq(mask)	__builtin_ctzl(mask)
 
-#define	bsrl(mask)	(__builtin_clz(mask) ^ 0x1f)
-
-#define	bsrq(mask)	(__builtin_clzl(mask) ^ 0x3f)
-
 static __inline void
 clflush(u_long addr)
 {

sys/arm64/iommu/smmu.c

@@ -310,15 +310,6 @@ smmu_write_ack(struct smmu_softc *sc, uint32_t reg,
 	return (0);
 }
 
-static inline int
-ilog2(long x)
-{
-
-	KASSERT(x > 0 && powerof2(x), ("%s: invalid arg %ld", __func__, x));
-
-	return (flsl(x) - 1);
-}
-
 static int
 smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q,
     uint32_t prod_off, uint32_t cons_off, uint32_t dwords)

sys/compat/linuxkpi/common/include/linux/log2.h

@@ -51,79 +51,6 @@ rounddown_pow_of_two(unsigned long x)
 	return (1UL << (flsl(x) - 1));
 }
 
-#define	ilog2(n)				\
-(						\
-	__builtin_constant_p(n) ? (		\
-		(n) < 1 ? -1 :			\
-		(n) & (1ULL << 63) ? 63 :	\
-		(n) & (1ULL << 62) ? 62 :	\
-		(n) & (1ULL << 61) ? 61 :	\
-		(n) & (1ULL << 60) ? 60 :	\
-		(n) & (1ULL << 59) ? 59 :	\
-		(n) & (1ULL << 58) ? 58 :	\
-		(n) & (1ULL << 57) ? 57 :	\
-		(n) & (1ULL << 56) ? 56 :	\
-		(n) & (1ULL << 55) ? 55 :	\
-		(n) & (1ULL << 54) ? 54 :	\
-		(n) & (1ULL << 53) ? 53 :	\
-		(n) & (1ULL << 52) ? 52 :	\
-		(n) & (1ULL << 51) ? 51 :	\
-		(n) & (1ULL << 50) ? 50 :	\
-		(n) & (1ULL << 49) ? 49 :	\
-		(n) & (1ULL << 48) ? 48 :	\
-		(n) & (1ULL << 47) ? 47 :	\
-		(n) & (1ULL << 46) ? 46 :	\
-		(n) & (1ULL << 45) ? 45 :	\
-		(n) & (1ULL << 44) ? 44 :	\
-		(n) & (1ULL << 43) ? 43 :	\
-		(n) & (1ULL << 42) ? 42 :	\
-		(n) & (1ULL << 41) ? 41 :	\
-		(n) & (1ULL << 40) ? 40 :	\
-		(n) & (1ULL << 39) ? 39 :	\
-		(n) & (1ULL << 38) ? 38 :	\
-		(n) & (1ULL << 37) ? 37 :	\
-		(n) & (1ULL << 36) ? 36 :	\
-		(n) & (1ULL << 35) ? 35 :	\
-		(n) & (1ULL << 34) ? 34 :	\
-		(n) & (1ULL << 33) ? 33 :	\
-		(n) & (1ULL << 32) ? 32 :	\
-		(n) & (1ULL << 31) ? 31 :	\
-		(n) & (1ULL << 30) ? 30 :	\
-		(n) & (1ULL << 29) ? 29 :	\
-		(n) & (1ULL << 28) ? 28 :	\
-		(n) & (1ULL << 27) ? 27 :	\
-		(n) & (1ULL << 26) ? 26 :	\
-		(n) & (1ULL << 25) ? 25 :	\
-		(n) & (1ULL << 24) ? 24 :	\
-		(n) & (1ULL << 23) ? 23 :	\
-		(n) & (1ULL << 22) ? 22 :	\
-		(n) & (1ULL << 21) ? 21 :	\
-		(n) & (1ULL << 20) ? 20 :	\
-		(n) & (1ULL << 19) ? 19 :	\
-		(n) & (1ULL << 18) ? 18 :	\
-		(n) & (1ULL << 17) ? 17 :	\
-		(n) & (1ULL << 16) ? 16 :	\
-		(n) & (1ULL << 15) ? 15 :	\
-		(n) & (1ULL << 14) ? 14 :	\
-		(n) & (1ULL << 13) ? 13 :	\
-		(n) & (1ULL << 12) ? 12 :	\
-		(n) & (1ULL << 11) ? 11 :	\
-		(n) & (1ULL << 10) ? 10 :	\
-		(n) & (1ULL <<  9) ?  9 :	\
-		(n) & (1ULL <<  8) ?  8 :	\
-		(n) & (1ULL <<  7) ?  7 :	\
-		(n) & (1ULL <<  6) ?  6 :	\
-		(n) & (1ULL <<  5) ?  5 :	\
-		(n) & (1ULL <<  4) ?  4 :	\
-		(n) & (1ULL <<  3) ?  3 :	\
-		(n) & (1ULL <<  2) ?  2 :	\
-		(n) & (1ULL <<  1) ?  1 :	\
-		(n) & (1ULL <<  0) ?  0 :	\
-		-1) :				\
-	(sizeof(n) <= 4) ?			\
-	fls((u32)(n)) - 1 : flsll((u64)(n)) - 1	\
-)
-
 #define	order_base_2(x) ilog2(roundup_pow_of_two(x))
 
 #endif	/* _LINUXKPI_LINUX_LOG2_H_ */

sys/dev/bxe/bxe.h

@@ -127,16 +127,6 @@
 #ifndef roundup
 #define roundup(x, y) ((((x) + ((y) - 1)) / (y)) * (y))
 #endif
-#ifndef ilog2
-static inline
-int bxe_ilog2(int x)
-{
-    int log = 0;
-    while (x >>= 1) log++;
-    return (log);
-}
-#define ilog2(x) bxe_ilog2(x)
-#endif
 
 #include "ecore_sp.h"
 

sys/dev/bxe/ecore_sp.h

@@ -159,7 +159,7 @@ typedef struct mtx ECORE_MUTEX_SPIN;
 #define ECORE_FREE(_s, _buf, _size) free(_buf, M_TEMP)
 
 #define SC_ILT(sc)  ((sc)->ilt)
-#define ILOG2(x)    bxe_ilog2(x)
+#define ILOG2(x)    ilog2(x)
 
 #define ECORE_ILT_ZALLOC(x, y, size)                                       \
     do {                                                                   \

sys/dev/cxgbe/osdep.h

@@ -130,14 +130,6 @@ typedef boolean_t bool;
 #define PCI_EXP_LNKSTA_NLW	PCIEM_LINK_STA_WIDTH
 #define PCI_EXP_DEVCTL2		PCIER_DEVICE_CTL2
 
-static inline int
-ilog2(long x)
-{
-	KASSERT(x > 0 && powerof2(x), ("%s: invalid arg %ld", __func__, x));
-
-	return (flsl(x) - 1);
-}
-
 static inline char *
 strstrip(char *s)
 {

sys/dev/enetc/enetc_hw.h

@@ -9,7 +9,6 @@
 
 #define BIT(x)	(1UL << (x))
 #define GENMASK(h, l)	(((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h))))
-#define ilog2(x)	(flsl(x) - 1)
 
 #define PCI_VENDOR_FREESCALE	0x1957
 

sys/dev/mana/gdma_util.h

@@ -170,15 +170,6 @@ find_first_zero_bit(const unsigned long *p, unsigned long max)
 	return (max);
 }
 
-static inline unsigned long
-ilog2(unsigned long x)
-{
-	unsigned long log = x;
-	while (x >>= 1)
-		log++;
-	return (log);
-}
-
 static inline unsigned long
 roundup_pow_of_two(unsigned long x)
 {

sys/dev/qat/qat/qat_ocf.c

@@ -517,7 +517,7 @@ qat_ocf_session_init(device_t dev,
 				  M_NOWAIT,
 				  0,
 				  ~1UL,
-				  1 << (bsrl(sessionCtxSize - 1) + 1),
+				  1 << (ilog2(sessionCtxSize - 1) + 1),
 				  0);
 	if (NULL == sessionCtx) {
 		device_printf(dev, "unable to allocate memory for session\n");

sys/i386/include/cpufunc.h

@@ -63,15 +63,6 @@ bsfl(u_int mask)
 	return (result);
 }
 
-static __inline __pure2 u_int
-bsrl(u_int mask)
-{
-	u_int	result;
-
-	__asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
-	return (result);
-}
-
 static __inline void
 clflush(u_long addr)
 {

sys/powerpc/booke/pmap.c

@@ -239,7 +239,6 @@ static __inline uint32_t tlb_calc_wimg(vm_paddr_t pa, vm_memattr_t ma);
 
 static vm_size_t tsize2size(unsigned int);
 static unsigned int size2tsize(vm_size_t);
-static unsigned long ilog2(unsigned long);
 
 static void set_mas4_defaults(void);
 

sys/powerpc/booke/pmap_32.c

@@ -116,7 +116,6 @@ static unsigned int kernel_ptbls;	/* Number of KVA ptbls. */
 #define	VM_MAPDEV_BASE	((vm_offset_t)VM_MAXUSER_ADDRESS + PAGE_SIZE)
 
 static void tid_flush(tlbtid_t tid);
-static unsigned long ilog2(unsigned long);
 
 /**************************************************************************/
 /* Page table management */
@@ -931,18 +930,6 @@ mmu_booke_quick_remove_page(vm_offset_t addr)
 /* TID handling */
 /**************************************************************************/
 
-/*
- * Return the largest uint value log such that 2^log <= num.
- */
-static unsigned long
-ilog2(unsigned long num)
-{
-	long lz;
-
-	__asm ("cntlzw %0, %1" : "=r" (lz) : "r" (num));
-	return (31 - lz);
-}
-
 /*
  * Invalidate all TLB0 entries which match the given TID. Note this is
  * dedicated for cases when invalidations should NOT be propagated to other

sys/powerpc/booke/pmap_64.c

@@ -125,7 +125,6 @@ static pte_t ****kernel_ptbl_root;
 #define	VM_MAPDEV_PA_MAX	0x4000000000000000 /* Don't encroach on DMAP */
 
 static void tid_flush(tlbtid_t tid);
-static unsigned long ilog2(unsigned long);
 
 /**************************************************************************/
 /* Page table management */
@@ -746,18 +745,6 @@ mmu_booke_quick_remove_page(vm_offset_t addr)
 /* TID handling */
 /**************************************************************************/
 
-/*
- * Return the largest uint value log such that 2^log <= num.
- */
-static unsigned long
-ilog2(unsigned long num)
-{
-	long lz;
-
-	__asm ("cntlzd %0, %1" : "=r" (lz) : "r" (num));
-	return (63 - lz);
-}
-
 /*
  * Invalidate all TLB0 entries which match the given TID. Note this is
  * dedicated for cases when invalidations should NOT be propagated to other

sys/sys/libkern.h

@@ -189,6 +189,112 @@ flsll(long long mask)
 	    8 * sizeof(mask) - __builtin_clzll((unsigned long long)mask));
 }
 
+static __inline __pure2 int
+ilog2_int(int n)
+{
+
+	MPASS(n != 0);
+	return (8 * sizeof(n) - 1 - __builtin_clz((u_int)n));
+}
+
+static __inline __pure2 int
+ilog2_long(long n)
+{
+
+	MPASS(n != 0);
+	return (8 * sizeof(n) - 1 - __builtin_clzl((u_long)n));
+}
+
+static __inline __pure2 int
+ilog2_long_long(long long n)
+{
+
+	MPASS(n != 0);
+	return (8 * sizeof(n) - 1 -
+	    __builtin_clzll((unsigned long long)n));
+}
+
+#define ilog2_var(n)				\
+	_Generic((n),				\
+	    default: ilog2_int,			\
+	    long: ilog2_long,			\
+	    unsigned long: ilog2_long,		\
+	    long long: ilog2_long_long,		\
+	    unsigned long long: ilog2_long_long	\
+	)(n)
+
+#define	ilog2(n)				\
+(						\
+	__builtin_constant_p(n) ? (		\
+	    (n) < 1 ? -1 :			\
+	    (n) & (1ULL << 63) ? 63 :		\
+	    (n) & (1ULL << 62) ? 62 :		\
+	    (n) & (1ULL << 61) ? 61 :		\
+	    (n) & (1ULL << 60) ? 60 :		\
+	    (n) & (1ULL << 59) ? 59 :		\
+	    (n) & (1ULL << 58) ? 58 :		\
+	    (n) & (1ULL << 57) ? 57 :		\
+	    (n) & (1ULL << 56) ? 56 :		\
+	    (n) & (1ULL << 55) ? 55 :		\
+	    (n) & (1ULL << 54) ? 54 :		\
+	    (n) & (1ULL << 53) ? 53 :		\
+	    (n) & (1ULL << 52) ? 52 :		\
+	    (n) & (1ULL << 51) ? 51 :		\
+	    (n) & (1ULL << 50) ? 50 :		\
+	    (n) & (1ULL << 49) ? 49 :		\
+	    (n) & (1ULL << 48) ? 48 :		\
+	    (n) & (1ULL << 47) ? 47 :		\
+	    (n) & (1ULL << 46) ? 46 :		\
+	    (n) & (1ULL << 45) ? 45 :		\
+	    (n) & (1ULL << 44) ? 44 :		\
+	    (n) & (1ULL << 43) ? 43 :		\
+	    (n) & (1ULL << 42) ? 42 :		\
+	    (n) & (1ULL << 41) ? 41 :		\
+	    (n) & (1ULL << 40) ? 40 :		\
+	    (n) & (1ULL << 39) ? 39 :		\
+	    (n) & (1ULL << 38) ? 38 :		\
+	    (n) & (1ULL << 37) ? 37 :		\
+	    (n) & (1ULL << 36) ? 36 :		\
+	    (n) & (1ULL << 35) ? 35 :		\
+	    (n) & (1ULL << 34) ? 34 :		\
+	    (n) & (1ULL << 33) ? 33 :		\
+	    (n) & (1ULL << 32) ? 32 :		\
+	    (n) & (1ULL << 31) ? 31 :		\
+	    (n) & (1ULL << 30) ? 30 :		\
+	    (n) & (1ULL << 29) ? 29 :		\
+	    (n) & (1ULL << 28) ? 28 :		\
+	    (n) & (1ULL << 27) ? 27 :		\
+	    (n) & (1ULL << 26) ? 26 :		\
+	    (n) & (1ULL << 25) ? 25 :		\
+	    (n) & (1ULL << 24) ? 24 :		\
+	    (n) & (1ULL << 23) ? 23 :		\
+	    (n) & (1ULL << 22) ? 22 :		\
+	    (n) & (1ULL << 21) ? 21 :		\
+	    (n) & (1ULL << 20) ? 20 :		\
+	    (n) & (1ULL << 19) ? 19 :		\
+	    (n) & (1ULL << 18) ? 18 :		\
+	    (n) & (1ULL << 17) ? 17 :		\
+	    (n) & (1ULL << 16) ? 16 :		\
+	    (n) & (1ULL << 15) ? 15 :		\
+	    (n) & (1ULL << 14) ? 14 :		\
+	    (n) & (1ULL << 13) ? 13 :		\
+	    (n) & (1ULL << 12) ? 12 :		\
+	    (n) & (1ULL << 11) ? 11 :		\
+	    (n) & (1ULL << 10) ? 10 :		\
+	    (n) & (1ULL <<  9) ?  9 :		\
+	    (n) & (1ULL <<  8) ?  8 :		\
+	    (n) & (1ULL <<  7) ?  7 :		\
+	    (n) & (1ULL <<  6) ?  6 :		\
+	    (n) & (1ULL <<  5) ?  5 :		\
+	    (n) & (1ULL <<  4) ?  4 :		\
+	    (n) & (1ULL <<  3) ?  3 :		\
+	    (n) & (1ULL <<  2) ?  2 :		\
+	    (n) & (1ULL <<  1) ?  1 :		\
+	    (n) & (1ULL <<  0) ?  0 :		\
+	    -1) :				\
+	ilog2_var(n)				\
+)
+
 #define	bitcount64(x)	__bitcount64((uint64_t)(x))
 #define	bitcount32(x)	__bitcount32((uint32_t)(x))
 #define	bitcount16(x)	__bitcount16((uint16_t)(x))