Implement pruning for 444-threephase

deployable-gce/src/main/kotlin/org/worldcubeassociation/tnoodle/deployable/gce/GoogleServerEnvironmentConfig.kt

@@ -1,10 +1,18 @@
 package org.worldcubeassociation.tnoodle.deployable.gce
 
+import com.google.cloud.storage.BlobId
+import com.google.cloud.storage.BlobInfo
+import com.google.cloud.storage.StorageOptions
 import org.worldcubeassociation.tnoodle.server.ServerEnvironmentConfig
 import java.io.File
 import java.io.File.separator
+import java.io.InputStream
+import java.io.OutputStream
+import java.nio.channels.Channels
 
 object GoogleServerEnvironmentConfig : ServerEnvironmentConfig {
+    val GCS_SERVICE by lazy { StorageOptions.getDefaultInstance().service }
+
     private val CONFIG_FILE = javaClass.getResourceAsStream("/version.tnoodle")
     private val CONFIG_DATA = CONFIG_FILE?.reader()?.readLines() ?: listOf()
 
@@ -23,9 +31,39 @@ object GoogleServerEnvironmentConfig : ServerEnvironmentConfig {
         get() = CONFIG_DATA.getOrNull(1)
             ?: "devel-TEMP"
 
+    override val usePruning = true
+
+    override fun pruningTableExists(tableName: String): Boolean {
+        val blobId = remotePruningBlob(tableName)
+        return GCS_SERVICE.get(blobId)?.exists() ?: false
+    }
+
+    override fun getPruningTableInput(tableName: String): InputStream {
+        val blobId = remotePruningBlob(tableName)
+        val blobReader = GCS_SERVICE.get(blobId).reader()
+
+        return Channels.newInputStream(blobReader)
+    }
+
+    override fun getPruningTableOutput(tableName: String): OutputStream {
+        val blobId = remotePruningBlob(tableName)
+        val blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build()
+
+        val blobWriter = GCS_SERVICE.create(blobInfo).writer()
+
+        return Channels.newOutputStream(blobWriter)
+    }
+
+    private fun remotePruningBlob(tableName: String) = BlobId.of(getCloudBucketName(), tableName)
+
+    private fun getCloudHostname() = System.getProperty("com.google.appengine.application.id")
+    fun getCloudBucketName() = "${getCloudHostname()}.appspot.com"
+
     fun overrideFontConfig() {
         if (File(FONT_CONFIG).exists()) {
             System.setProperty(FONT_CONFIG_PROPERTY, FONT_CONFIG)
         }
     }
+
+    override fun createLocalPruningCache() = overrideFontConfig() // FIXME do we want this here implicitly?
 }

deployable-jar/src/main/kotlin/org/worldcubeassociation/tnoodle/deployable/jar/config/LocalServerEnvironmentConfig.kt

@@ -2,7 +2,11 @@ package org.worldcubeassociation.tnoodle.deployable.jar.config
 
 import org.worldcubeassociation.tnoodle.server.ServerEnvironmentConfig
 import org.worldcubeassociation.tnoodle.deployable.jar.server.WebServerUtils.DEVEL_VERSION
+import org.worldcubeassociation.tnoodle.deployable.jar.server.WebServerUtils.PRUNING_FOLDER
 import org.worldcubeassociation.tnoodle.deployable.jar.server.WebServerUtils
+import org.worldcubeassociation.tnoodle.deployable.jar.server.WebServerUtils.jarFile
+import java.io.File
+import java.io.FileNotFoundException
 
 object LocalServerEnvironmentConfig : ServerEnvironmentConfig {
     override val projectName
@@ -12,4 +16,46 @@ object LocalServerEnvironmentConfig : ServerEnvironmentConfig {
     override val projectVersion
         get() = this::class.java.getPackage()?.implementationVersion
             ?: DEVEL_VERSION
+
+    override val usePruning: Boolean
+        get() = System.getenv("CI") != null
+
+    private fun getPruningTableCache(assertExists: Boolean = true): File {
+        val baseDir = File(WebServerUtils.programDirectory, PRUNING_FOLDER)
+
+        // Each version of TNoodle extracts its pruning tables
+        // to its own subdirectory of PRUNING_FOLDER
+        val file = baseDir.takeIf { projectVersion == DEVEL_VERSION }
+            ?: File(baseDir, projectVersion)
+
+        if (assertExists && !file.isDirectory) {
+            throw FileNotFoundException("${file.absolutePath} does not exist, or is not a directory")
+        }
+
+        return file
+    }
+
+    override fun pruningTableExists(tableName: String) = localPruningFile(tableName).exists()
+
+    override fun getPruningTableInput(tableName: String) = localPruningFile(tableName).inputStream()
+
+    override fun getPruningTableOutput(tableName: String) =
+        localPruningFile(tableName).takeIf { it.parentFile.isDirectory || it.parentFile.mkdirs() }?.outputStream()
+            ?: error("Unable to create pruning file for table '$tableName'")
+
+    private fun localPruningFile(tableName: String) = File(getPruningTableCache(false), "$tableName.prun")
+
+    override fun createLocalPruningCache() {
+        if (jarFile != null) {
+            val pruningTableDirectory = getPruningTableCache(false)
+
+            if (pruningTableDirectory.isDirectory) {
+                // If the pruning table folder already exists, we don't bother re-extracting the
+                // files.
+                return
+            }
+
+            assert(pruningTableDirectory.mkdirs())
+        }
+    }
 }

deployable-jar/src/main/kotlin/org/worldcubeassociation/tnoodle/deployable/jar/server/WebServerUtils.kt

@@ -6,7 +6,8 @@ import java.nio.file.Files
 import java.nio.file.StandardCopyOption
 
 object WebServerUtils {
-    val DEVEL_VERSION = "devel-TEMP"
+    const val DEVEL_VERSION = "devel-TEMP"
+    const val PRUNING_FOLDER = "pruning-tables"
 
     // Classes that are part of a web app were loaded with the
     // servlet container's classloader, so we can't necessarily
@@ -49,6 +50,12 @@ object WebServerUtils {
     val jarFile: File?
         get() = jarFileOrDirectory.takeIf { it.isFile }
 
+    val programDirectory: File
+        get() {
+            val programDirectory = jarFileOrDirectory
+            return programDirectory.takeUnless { it.isFile } ?: programDirectory.parentFile
+        }
+
     fun copyFile(sourceFile: File, destFile: File) =
         Files.copy(sourceFile.toPath(), destFile.toPath(), StandardCopyOption.REPLACE_EXISTING)
 }

gradle/libs.versions.toml

@@ -6,6 +6,7 @@ junit-jupiter = "5.10.1"
 batik = "1.17"
 itext7 = "8.0.2"
 logback = "1.3.14"
+tnoodle-lib = "0.19.2"
 
 nodejs = "20.10.0"
 
@@ -47,7 +48,8 @@ proguard-gradle = { module = "com.guardsquare:proguard-gradle", version = "7.4.1
 wca-i18n = { module = "com.github.thewca:wca_i18n", version = "0.4.3" }
 google-appengine-gradle = { module = "com.google.cloud.tools:appengine-gradle-plugin", version = "2.5.0" }
 google-cloud-storage = { module = "com.google.cloud:google-cloud-storage", version = "2.30.1" }
-tnoodle-scrambles = { module = "org.worldcubeassociation.tnoodle:lib-scrambles", version = "0.19.2" }
+tnoodle-scrambles = { module = "org.worldcubeassociation.tnoodle:lib-scrambles", version.ref = "tnoodle-lib" }
+tnoodle-scrambler-threephase = { module = "org.worldcubeassociation.tnoodle:scrambler-threephase", version.ref = "tnoodle-lib" }
 apache-commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.14.0" }
 kotless-ktor = { module = "io.kotless:ktor-lang", version.ref = "kotless" }
 mockk = { module = "io.mockk:mockk", version = "1.13.8" }

server/build.gradle.kts

@@ -33,6 +33,7 @@ dependencies {
     api(libs.kotlinx.serialization.json)
     api(libs.tnoodle.scrambles)
 
+    implementation(libs.tnoodle.scrambler.threephase)
     implementation(libs.zip4j)
     implementation(libs.itextpdf)
     implementation(libs.itext7)

server/src/main/kotlin/org/worldcubeassociation/tnoodle/server/ServerEnvironmentConfig.kt

@@ -1,9 +1,21 @@
 package org.worldcubeassociation.tnoodle.server
 
+import java.io.InputStream
+import java.io.OutputStream
+
 interface ServerEnvironmentConfig {
     val projectName: String
     val projectVersion: String
 
     val title
         get() = "$projectName-$projectVersion"
+
+    val usePruning: Boolean
+
+    fun pruningTableExists(tableName: String): Boolean
+
+    fun getPruningTableInput(tableName: String): InputStream
+    fun getPruningTableOutput(tableName: String): OutputStream
+
+    fun createLocalPruningCache()
 }

server/src/main/kotlin/org/worldcubeassociation/tnoodle/server/TNoodleServer.kt

@@ -1,5 +1,6 @@
 package org.worldcubeassociation.tnoodle.server
 
+import cs.threephase.Tools
 import io.ktor.http.*
 import io.ktor.serialization.kotlinx.json.*
 import io.ktor.server.application.*
@@ -26,9 +27,13 @@ import org.worldcubeassociation.tnoodle.server.serial.JsonConfig
 import org.worldcubeassociation.tnoodle.server.routing.frontend.ApplicationDataHandler
 import org.worldcubeassociation.tnoodle.server.routing.frontend.PuzzleDrawingHandler
 import org.worldcubeassociation.tnoodle.server.routing.frontend.WcifDataHandler
+import java.io.DataInputStream
+import java.io.DataOutputStream
 
-class TNoodleServer(val environmentConfig: ServerEnvironmentConfig = TNoodleServer) : ApplicationHandler {
+class TNoodleServer(val environmentConfig: ServerEnvironmentConfig) : ApplicationHandler {
     override fun spinUp(app: Application) {
+        initPruning()
+
         val versionHandler = VersionHandler(environmentConfig)
         val wcifHandler = WcifHandler(environmentConfig)
 
@@ -86,10 +91,23 @@ class TNoodleServer(val environmentConfig: ServerEnvironmentConfig = TNoodleServ
         }
     }
 
-    companion object : ServerEnvironmentConfig {
+    private fun initPruning() {
+        if (environmentConfig.usePruning) {
+            if (environmentConfig.pruningTableExists(THREEPHASE_PRUNING)) {
+                DataInputStream(environmentConfig.getPruningTableInput(THREEPHASE_PRUNING)).use {
+                    Tools.initFrom(it)
+                }
+            } else {
+                DataOutputStream(environmentConfig.getPruningTableOutput(THREEPHASE_PRUNING)).use {
+                    Tools.saveTo(it)
+                }
+            }
+        }
+    }
+
+    companion object {
         const val KILL_URL = "/kill/tnoodle/now"
 
-        override val projectName = "TNoodle-BACKEND"
-        override val projectVersion = "devel-TEMP"
+        const val THREEPHASE_PRUNING = "444-threephase"
     }
 }