diff --git a/arch/arm/configs/socfpga_adi_defconfig b/arch/arm/configs/socfpga_adi_defconfig
index 530c5f311af0fd..04b44ba3ffe5c0 100644
--- a/arch/arm/configs/socfpga_adi_defconfig
+++ b/arch/arm/configs/socfpga_adi_defconfig
@@ -111,11 +111,6 @@ CONFIG_MFD_STMPE=y
 CONFIG_REGULATOR=y
 CONFIG_REGULATOR_FIXED_VOLTAGE=y
 CONFIG_FB_ALTERA_VIP=y
-CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
-CONFIG_USB_STORAGE=y
-CONFIG_USB_DWC2=y
-CONFIG_NOP_USB_XCEIV=y
-CONFIG_USB_GADGET=y
 CONFIG_USB_MASS_STORAGE=m
 CONFIG_MMC=y
 CONFIG_MMC_DW=y
diff --git a/arch/arm/configs/zynq_xcomm_adv7511_defconfig b/arch/arm/configs/zynq_xcomm_adv7511_defconfig
index 0bef5134603aa7..a678e7c945b8fd 100644
--- a/arch/arm/configs/zynq_xcomm_adv7511_defconfig
+++ b/arch/arm/configs/zynq_xcomm_adv7511_defconfig
@@ -145,8 +145,6 @@ CONFIG_XILINX_WATCHDOG=y
 CONFIG_CADENCE_WATCHDOG=y
 CONFIG_REGULATOR=y
 CONFIG_REGULATOR_FIXED_VOLTAGE=y
-CONFIG_MEDIA_USB_SUPPORT=y
-CONFIG_USB_VIDEO_CLASS=y
 CONFIG_USB_GSPCA=y
 CONFIG_VIDEO_IMAGEON_BRIDGE=y
 # CONFIG_MEDIA_SUBDRV_AUTOSELECT is not set
@@ -157,7 +155,6 @@ CONFIG_LOGO=y
 # CONFIG_SND_DRIVERS is not set
 # CONFIG_SND_ARM is not set
 # CONFIG_SND_SPI is not set
-# CONFIG_SND_USB is not set
 CONFIG_HIDRAW=y
 CONFIG_HID_A4TECH=y
 CONFIG_HID_ACRUX=y
@@ -212,22 +209,7 @@ CONFIG_THRUSTMASTER_FF=y
 CONFIG_HID_ZEROPLUS=y
 CONFIG_ZEROPLUS_FF=y
 CONFIG_HID_ZYDACRON=y
-CONFIG_USB_HIDDEV=y
-CONFIG_USB_OTG=y
-CONFIG_USB_EHCI_HCD=y
 # CONFIG_USB_EHCI_TT_NEWSCHED is not set
-CONFIG_USB_STORAGE=y
-CONFIG_USB_CHIPIDEA=y
-CONFIG_USB_CHIPIDEA_UDC=y
-CONFIG_USB_CHIPIDEA_HOST=y
-CONFIG_USB_SERIAL_GENERIC=y
-CONFIG_USB_SERIAL_FTDI_SIO=y
-CONFIG_USB_ULPI=y
-CONFIG_USB_GADGET=y
-CONFIG_USB_GADGET_XILINX=y
-CONFIG_USB_CONFIGFS=y
-CONFIG_USB_CONFIGFS_ACM=y
-CONFIG_USB_CONFIGFS_F_FS=y
 CONFIG_MMC=y
 CONFIG_MMC_SDHCI=y
 CONFIG_MMC_SDHCI_PLTFM=y
diff --git a/arch/arm64/configs/adi_zynqmp_defconfig b/arch/arm64/configs/adi_zynqmp_defconfig
index fa802bfe92a02a..e0cee42dcc1856 100644
--- a/arch/arm64/configs/adi_zynqmp_defconfig
+++ b/arch/arm64/configs/adi_zynqmp_defconfig
@@ -189,7 +189,6 @@ CONFIG_SMSC_PHY=y
 CONFIG_STE10XP=y
 CONFIG_VITESSE_PHY=y
 CONFIG_XILINX_GMII2RGMII=y
-CONFIG_USB_USBNET=y
 CONFIG_WL18XX=y
 CONFIG_WLCORE_SPI=y
 CONFIG_WLCORE_SDIO=y
@@ -233,8 +232,6 @@ CONFIG_REGULATOR=y
 CONFIG_REGULATOR_FIXED_VOLTAGE=y
 CONFIG_REGULATOR_GPIO=y
 CONFIG_REGULATOR_TPS65086=y
-CONFIG_MEDIA_USB_SUPPORT=y
-CONFIG_USB_VIDEO_CLASS=y
 CONFIG_VIDEO_XILINX=y
 CONFIG_VIDEO_XILINX_AXI4S_SWITCH=y
 CONFIG_VIDEO_XILINX_CFA=y
@@ -273,28 +270,12 @@ CONFIG_FB_XILINX=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
 # CONFIG_SND_DRIVERS is not set
 # CONFIG_SND_PCI is not set
-CONFIG_SND_USB_AUDIO=y
 CONFIG_SND_SOC_XILINX_DP=y
 CONFIG_SND_SOC_XILINX_AUDIO_FORMATTER=y
 CONFIG_SND_SOC_XILINX_SDI=y
 CONFIG_SND_SOC_XILINX_I2S=y
 CONFIG_SND_SOC_XILINX_SPDIF=y
 CONFIG_SND_SOC_XILINX_PL_SND_CARD=y
-CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
-# CONFIG_USB_DEFAULT_PERSIST is not set
-CONFIG_USB_OTG=y
-CONFIG_USB_OTG_FSM=y
-CONFIG_USB_XHCI_HCD=y
-CONFIG_USB_STORAGE=y
-CONFIG_USB_UAS=y
-CONFIG_USB_DWC3=y
-CONFIG_USB_GADGET=y
-CONFIG_USB_GADGET_XILINX=y
-CONFIG_USB_CONFIGFS=y
-CONFIG_USB_CONFIGFS_ACM=y
-CONFIG_USB_CONFIGFS_EEM=y
-CONFIG_USB_CONFIGFS_MASS_STORAGE=y
-CONFIG_USB_CONFIGFS_F_FS=y
 CONFIG_USB_ETH=m
 CONFIG_USB_ETH_EEM=y
 CONFIG_USB_MASS_STORAGE=m
@@ -346,7 +327,6 @@ CONFIG_ZYNQMP_R5_REMOTEPROC=m
 CONFIG_XILINX_VCU=m
 CONFIG_XILINX_AMS=y
 CONFIG_XILINX_INTC=y
-CONFIG_RESET_CONTROLLER=y
 CONFIG_RAS=y
 CONFIG_ANDROID=y
 CONFIG_NVMEM_ZYNQMP=y