🔨 batch insert 1000 skulls at a time, initial inserts are much faster…

… now. re-run a silent download to compare current heads with new ones to see if any is missing then insert. Closes #29

Took 38 minutes

src/main/java/ca/tweetzy/skulls/Skulls.java

@@ -92,7 +92,12 @@ protected void onFlight() {
 		this.guiManager.init();
 
 		// command
-		this.commandManager.registerCommandDynamically(new SkullsCommand()).addSubCommands(new SearchCommand(), new PlayerHeadCommand(), new GiveCommand(), new InspectCommand());
+		this.commandManager.registerCommandDynamically(new SkullsCommand()).addSubCommands(
+				new SearchCommand(),
+				new PlayerHeadCommand(),
+				new GiveCommand(),
+				new InspectCommand()
+		);
 
 		// events
 		getServer().getPluginManager().registerEvents(new PlayerJoinQuitListener(), this);

src/main/java/ca/tweetzy/skulls/database/DataManager.java

@@ -56,8 +56,15 @@ public DataManager(DatabaseConnector databaseConnector, Plugin plugin) {
 	}
 
 	public void insertSkulls(Collection<Skull> skulls) {
+		final int batchSize = 1000;
 		this.runAsync(() -> this.databaseConnector.connect(connection -> {
-			PreparedStatement statement = connection.prepareStatement("INSERT OR IGNORE INTO " + this.getTablePrefix() + "skull(id, name, category, texture, tags, price, blocked) VALUES(?, ?, ?, ?, ?, ?, ?)");
+			connection.setAutoCommit(false);
+
+			PreparedStatement statement = connection.prepareStatement(
+					"INSERT OR IGNORE INTO " + this.getTablePrefix() + "skull(id, name, category, texture, tags, price, blocked) VALUES(?, ?, ?, ?, ?, ?, ?)"
+			);
+
+			int count = 0;
 			for (Skull skull : skulls) {
 				statement.setInt(1, skull.getId());
 				statement.setString(2, skull.getName());
@@ -67,9 +74,16 @@ public void insertSkulls(Collection<Skull> skulls) {
 				statement.setDouble(6, skull.getPrice());
 				statement.setBoolean(7, skull.isBlocked());
 				statement.addBatch();
+
+				if (++count % batchSize == 0) {
+					statement.executeBatch(); // Execute batch every batchSize
+				}
 			}
 
 			statement.executeBatch();
+			connection.commit();
+			statement.close();
+
 			Skulls.getSkullManager().setDownloading(false);
 			Common.log("&r&aFinished inserting all heads into the data file!");
 		}));

src/main/java/ca/tweetzy/skulls/manager/SkullManager.java

@@ -162,6 +162,30 @@ public void removePlacedSkull(@NonNull final PlacedSkull placedSkull) {
 	 * For internal use only
 	 */
 
+	private void checkAndFixDatabase() {
+		Bukkit.getServer().getScheduler().runTaskAsynchronously(Skulls.getInstance(), () -> {
+			final Set<Skull> heads = new HashSet<>();
+
+			Common.log("&r&aRunning database check :)");
+
+			for (BaseCategory value : BaseCategory.values()) {
+				final List<Skull> downloaded = downloadHeadCategory(value, true);
+				downloaded.forEach(skull -> {
+					if (this.skulls.contains(skull)) return;
+					heads.add(skull);
+				});
+			}
+
+			if (this.skulls.size() < heads.size()) {
+				Common.log("&r&eFound some missing heads, downloading/inserting them now!");
+
+				this.skulls.addAll(heads);
+				this.idList.addAll(heads.stream().map(Skull::getId).toList());
+				Skulls.getDataManager().insertSkulls(new ArrayList<>(heads));
+			}
+		});
+	}
+
 	public void downloadHeads() {
 		setDownloading(true);
 		Bukkit.getServer().getScheduler().runTaskAsynchronously(Skulls.getInstance(), () -> {
@@ -170,15 +194,16 @@ public void downloadHeads() {
 			Common.log("&r&aBeginning initial download, it may take some time to insert all the skulls into the data file!");
 
 			for (BaseCategory value : BaseCategory.values()) {
-				heads.addAll(downloadHeadCategory(value));
+				heads.addAll(downloadHeadCategory(value, false));
 			}
 
 			this.skulls.addAll(heads);
-			this.idList.addAll(heads.stream().map(Skull::getId).collect(Collectors.toList()));
+			this.idList.addAll(heads.stream().map(Skull::getId).toList());
 			Skulls.getDataManager().insertSkulls(heads);
 		});
 	}
 
+
 	public List<History> downloadHistories() {
 		final List<History> histories = new ArrayList<>();
 		try {
@@ -199,7 +224,7 @@ public List<History> downloadHistories() {
 		return histories;
 	}
 
-	private List<Skull> downloadHeadCategory(@NonNull final BaseCategory category) {
+	public List<Skull> downloadHeadCategory(@NonNull final BaseCategory category, boolean silent) {
 		final List<Skull> heads = new ArrayList<>();
 		try {
 			long start = System.nanoTime();
@@ -221,9 +246,11 @@ private List<Skull> downloadHeadCategory(@NonNull final BaseCategory category) {
 
 			});
 
-			Common.log("&aDownloaded &e" + heads.size() + " &askulls for category &e" + category.getName() + "&a in &f" + String.format("%,.3f", (System.nanoTime() - start) / 1e+6) + "&ems");
+			if (!silent)
+				Common.log("&aDownloaded &e" + heads.size() + " &askulls for category &e" + category.getName() + "&a in &f" + String.format("%,.3f", (System.nanoTime() - start) / 1e+6) + "&ems");
 		} catch (Exception e) {
-			Common.log("&cTweetzy.ca's api is currently unavailable, you can try again shortly.");
+			if (!silent)
+				Common.log("&cTweetzy.ca's api is currently unavailable, you can try again shortly.");
 		}
 
 		return heads;
@@ -289,14 +316,15 @@ public void load() {
 			}
 
 			this.skulls.addAll(all);
-			this.idList.addAll(all.stream().map(Skull::getId).collect(Collectors.toList()));
+			this.idList.addAll(all.stream().map(Skull::getId).toList());
 
 			if (this.skulls.isEmpty()) {
 				Common.log("&cCould not find any skulls, attempting to redownload them!");
 				downloadHeads();
 			} else {
 				Common.log("&aLoaded &e" + this.skulls.size() + " &askulls in &f" + String.format("%,.3f", (System.nanoTime() - start) / 1e+6) + "&ams");
 				setLoading(false);
+				checkAndFixDatabase();
 			}
 		});