Implement pakku fetch --shelve flag & remove sync subpath auto-detect

src/commonMain/kotlin/teksturepako/pakku/Version.kt

@@ -2,4 +2,4 @@
 
 package teksturepako.pakku
 
-const val VERSION = "0.23.0"
+const val VERSION = "0.24.1"

src/commonMain/kotlin/teksturepako/pakku/api/actions/fetch/DeleteOldFiles.kt

@@ -8,30 +8,61 @@ import kotlinx.coroutines.channels.consumeEach
 import kotlinx.coroutines.channels.produce
 import teksturepako.pakku.api.actions.errors.ActionError
 import teksturepako.pakku.api.actions.errors.DirectoryNotEmpty
+import teksturepako.pakku.api.actions.sync.detectProjects
 import teksturepako.pakku.api.data.ConfigFile
+import teksturepako.pakku.api.data.Dirs
 import teksturepako.pakku.api.data.LockFile
 import teksturepako.pakku.api.data.workingPath
 import teksturepako.pakku.api.overrides.ProjectOverride
+import teksturepako.pakku.api.platforms.Platform
 import teksturepako.pakku.api.projects.ProjectFile
 import teksturepako.pakku.api.projects.ProjectType
 import teksturepako.pakku.io.*
 import java.io.File
 import java.nio.file.Path
 import kotlin.io.path.*
 
+enum class DeletionActionType(val result: String)
+{
+    DELETE("deleted"), SHELF("shelved")
+}
+
 @OptIn(ExperimentalCoroutinesApi::class)
 suspend fun deleteOldFiles(
     onError: suspend (error: ActionError) -> Unit,
-    onSuccess: suspend (file: Path) -> Unit,
+    onSuccess: suspend (file: Path, actionType: DeletionActionType) -> Unit,
     projectFiles: List<ProjectFile>,
     projectOverrides: Set<ProjectOverride>,
     lockFile: LockFile,
-    configFile: ConfigFile?
+    configFile: ConfigFile?,
+    platforms: List<Platform>,
+    shelve: Boolean = false,
 ) = coroutineScope {
 
     val defaultIgnoredPaths = listOf("saves", "screenshots")
     val allowedExtensions = listOf(".jar", ".zip", ".jar.meta")
 
+    val detectedProjects = async {
+        detectProjects(lockFile, configFile, platforms)
+            .flatMap { project -> project.files }
+            .map { projectFile ->
+                async x@ {
+                    val parentProject = projectFile.getParentProject(lockFile) ?: return@x null
+
+                    val path = projectFile.getPath(parentProject, configFile)
+
+                    readPathBytesToResult(path)
+                        .onFailure { onError(it) }
+                        .get()?.let { path to it }
+                }
+            }
+            .awaitAll()
+            .filterNotNull()
+            .associate { (path, bytes) ->
+                path.absolute() to createHash("sha1", bytes)
+            }
+    }
+
     val fileHashes = async { projectFiles
         .map { projectFile ->
             async x@ {
@@ -81,22 +112,40 @@ suspend fun deleteOldFiles(
 
                     if (path.absolute() !in fileHashes.await().keys || hash !in fileHashes.await().values)
                     {
-                        send(path)
+                        send(path to hash)
                     }
                 }
             }
         }
     }
 
-    channel.consumeEach { path ->
+    channel.consumeEach { (path, hash) ->
         launch(Dispatchers.IO) {
-            path.tryToResult {
-                if (defaultIgnoredPaths.none { ignored -> ignored in it.pathString })
-                {
-                    it.deleteIfExists()
+            if (defaultIgnoredPaths.any { ignored -> ignored in path.pathString }) return@launch
+
+            if (shelve)
+            {
+                if (path.absolute() in detectedProjects.await().keys
+                    || hash in detectedProjects.await().values
+                    || path.isDirectory()) return@launch
+
+                path.tryToResult {
+                    Dirs.shelfDir.createDirectories()
+                    val newFile = Path(Dirs.shelfDir.pathString, it.fileName.pathString)
+                    it.moveTo(newFile)
+                }.onSuccess {
+                    onSuccess(path, DeletionActionType.SHELF)
+                }.onFailure { error ->
+                    if (error !is DirectoryNotEmpty) onError(error)
                 }
+
+                return@launch
+            }
+
+            path.tryToResult {
+                it.deleteIfExists()
             }.onSuccess {
-                onSuccess(path)
+                onSuccess(path, DeletionActionType.DELETE)
             }.onFailure { error ->
                 if (error !is DirectoryNotEmpty) onError(error)
             }

src/commonMain/kotlin/teksturepako/pakku/api/actions/sync/DetectProjects.kt

@@ -0,0 +1,102 @@
+package teksturepako.pakku.api.actions.sync
+
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
+import teksturepako.pakku.api.actions.update.combineProjects
+import teksturepako.pakku.api.data.ConfigFile
+import teksturepako.pakku.api.data.LockFile
+import teksturepako.pakku.api.data.workingPath
+import teksturepako.pakku.api.platforms.CurseForge
+import teksturepako.pakku.api.platforms.Modrinth
+import teksturepako.pakku.api.platforms.Platform
+import teksturepako.pakku.api.projects.Project
+import teksturepako.pakku.api.projects.ProjectType
+import teksturepako.pakku.api.projects.inheritPropertiesFrom
+import teksturepako.pakku.debugIfNotEmpty
+import teksturepako.pakku.io.createHash
+import teksturepako.pakku.io.readPathBytesOrNull
+import teksturepako.pakku.io.tryOrNull
+import teksturepako.pakku.toPrettyString
+import java.io.File
+import kotlin.io.path.Path
+import kotlin.io.path.isDirectory
+import kotlin.io.path.notExists
+import kotlin.io.path.pathString
+
+suspend fun detectProjects(
+    lockFile: LockFile,
+    configFile: ConfigFile?,
+    platforms: List<Platform>,
+): Set<Project> = coroutineScope {
+
+    val defaultIgnoredPaths = listOf("saves", "screenshots")
+    val allowedExtensions = listOf(".jar", ".zip")
+
+    val files = ProjectType.entries
+        .filterNot { it == ProjectType.WORLD }
+        .mapNotNull { projectType ->
+            val prjTypeDir = Path(workingPath, projectType.getPathString(configFile))
+            if (prjTypeDir.notExists() || defaultIgnoredPaths.any { it in prjTypeDir.pathString }) return@mapNotNull null
+
+            prjTypeDir
+        }
+        .mapNotNull { dir ->
+            dir.tryOrNull { path ->
+                path.toFile().walkTopDown().mapNotNull { file: File ->
+                    file.toPath().takeIf { it != dir }
+                }
+            }
+        }
+        .flatMap { pathSequence ->
+            pathSequence.toSet().map { path ->
+                async {
+                    if (path.isDirectory() || allowedExtensions.none { path.pathString.endsWith(it) }) return@async null
+
+                    val bytes = readPathBytesOrNull(path) ?: return@async null
+
+                    Triple(path, bytes, createHash("sha1", bytes))
+                }
+            }
+        }
+        .awaitAll()
+        .filterNotNull()
+        .debugIfNotEmpty {
+            println(it.map { (path, _) -> path.pathString }.toPrettyString())
+        }
+
+    val cfProjectsDeferred = async {
+        if (CurseForge in platforms)
+        {
+            CurseForge.requestMultipleProjectsWithFilesFromBytes(lockFile.getMcVersions(), files.map { it.second })
+                .inheritPropertiesFrom(configFile)
+        }
+        else mutableSetOf()
+    }
+
+    val mrProjectsDeferred = async {
+        if (Modrinth in platforms)
+        {
+            Modrinth.requestMultipleProjectsWithFilesFromHashes(files.map { it.third }, "sha1")
+                .inheritPropertiesFrom(configFile)
+        }
+        else mutableSetOf()
+    }
+
+    val deferredPlatformsToProjects = async {
+        listOf(CurseForge to cfProjectsDeferred.await(), Modrinth to mrProjectsDeferred.await())
+    }
+
+    val detectedProjects = deferredPlatformsToProjects.await()
+        .fold(deferredPlatformsToProjects.await().flatMap { it.second }) { accProjects, (platform, platformProjects) ->
+            accProjects.map { accProject ->
+                platformProjects.find { it isAlmostTheSameAs accProject }
+                    ?.let { newProject -> combineProjects(accProject, newProject, platform.serialName, 1) }
+                    ?: accProject
+            }
+        }
+        .distinctBy { it.files }
+        .toSet()
+
+    return@coroutineScope detectedProjects
+}

src/commonMain/kotlin/teksturepako/pakku/api/actions/sync/SyncProjects.kt

@@ -1,27 +1,14 @@
 package teksturepako.pakku.api.actions.sync
 
-import kotlinx.coroutines.async
-import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.runBlocking
 import teksturepako.pakku.api.actions.update.combineProjects
 import teksturepako.pakku.api.data.ConfigFile
 import teksturepako.pakku.api.data.LockFile
-import teksturepako.pakku.api.data.workingPath
-import teksturepako.pakku.api.platforms.*
+import teksturepako.pakku.api.platforms.GitHub
+import teksturepako.pakku.api.platforms.Multiplatform
+import teksturepako.pakku.api.platforms.Platform
 import teksturepako.pakku.api.projects.Project
-import teksturepako.pakku.api.projects.ProjectType
 import teksturepako.pakku.api.projects.containNotProject
-import teksturepako.pakku.api.projects.inheritPropertiesFrom
-import teksturepako.pakku.debug
-import teksturepako.pakku.debugIfNotEmpty
-import teksturepako.pakku.io.createHash
-import teksturepako.pakku.io.readAndCreateSha1FromBytes
-import teksturepako.pakku.io.readPathBytesOrNull
-import teksturepako.pakku.io.tryOrNull
-import teksturepako.pakku.toPrettyString
-import java.io.File
-import kotlin.io.path.*
 
 data class SyncResult(
     val added: Set<Project>,
@@ -35,126 +22,18 @@ suspend fun syncProjects(
     platforms: List<Platform>,
 ): SyncResult = coroutineScope {
 
-    val defaultIgnoredPaths = listOf("saves", "screenshots")
-    val allowedExtensions = listOf(".jar", ".zip")
-
-    val files = ProjectType.entries
-        .filterNot { it == ProjectType.WORLD }
-        .mapNotNull { projectType ->
-            val prjTypeDir = Path(workingPath, projectType.getPathString(configFile))
-            if (prjTypeDir.notExists() || defaultIgnoredPaths.any { it in prjTypeDir.pathString }) return@mapNotNull null
-
-            prjTypeDir
-        }
-        .mapNotNull { dir ->
-            dir.tryOrNull { path ->
-                path.toFile().walkTopDown().mapNotNull { file: File ->
-                    file.toPath().takeIf { it != dir }
-                }
-            }
-        }
-        .flatMap { pathSequence ->
-            pathSequence.toSet().map { path ->
-                async {
-                    if (path.isDirectory() || allowedExtensions.none { path.pathString.endsWith(it) }) return@async null
-
-                    val bytes = readPathBytesOrNull(path) ?: return@async null
-
-                    path to bytes
-                }
-            }
-        }
-        .awaitAll()
-        .filterNotNull()
-        .debugIfNotEmpty {
-            println(it.map { (path, _) -> path.pathString }.toPrettyString())
-        }
-
-    suspend fun List<Project>.withFoundSubpaths(): List<Project> = this.map { project ->
-        val subpath = files.mapNotNull { (path, bytes) ->
-            val projectFile = project.files
-                .map { file ->
-                    async {
-                        val hash = file.hashes?.get("sha1")
-                            ?: file.getPath(project, configFile).readAndCreateSha1FromBytes()
-                        file to hash
-                    }
-                }
-                .awaitAll()
-                .find { (_, hash) ->
-                    hash == createHash("sha1", bytes)
-                }
-                ?.first
-                ?: return@mapNotNull null
-
-            debug {
-                println("Found subpath for ${projectFile.fileName}")
-            }
-
-            val subpath = path.absolute().invariantSeparatorsPathString
-                .substringAfter(project.type.getPathString(configFile) + "/")
-                .substringBefore(projectFile.fileName)
-                .removeSuffix("/")
-
-            subpath.ifBlank { return@mapNotNull null }
-        }.firstOrNull()
-
-        debug {
-            println("Subpath: $subpath")
-        }
-
-        if (subpath != null)
-        {
-            project.copy().apply { setSubpath(subpath) }
-        }
-        else project
-    }
-
-    val cfProjectsDeferred = async {
-        if (CurseForge in platforms)
-        {
-            CurseForge.requestMultipleProjectsWithFilesFromBytes(lockFile.getMcVersions(), files.map { it.second })
-                .inheritPropertiesFrom(configFile)
-        }
-        else mutableSetOf()
-    }
-
-    val mrProjectsDeferred = async {
-        if (Modrinth in platforms)
-        {
-            Modrinth.requestMultipleProjectsWithFilesFromHashes(files.map { createHash("sha1", it.second) }, "sha1")
-                .inheritPropertiesFrom(configFile)
-        }
-        else mutableSetOf()
-    }
-
-    val deferredPlatformsToProjects = async {
-        listOf(CurseForge to cfProjectsDeferred.await(), Modrinth to mrProjectsDeferred.await())
-    }
-
-    val detectedProjects = deferredPlatformsToProjects.await()
-        .fold(deferredPlatformsToProjects.await().flatMap { it.second }) { accProjects, (platform, platformProjects) ->
-            accProjects.map { accProject ->
-                platformProjects.find { it isAlmostTheSameAs accProject }
-                    ?.let { newProject -> combineProjects(accProject, newProject, platform.serialName, 1) }
-                    ?: accProject
-            }
-        }
-        .distinctBy { it.files }
-        .sortedBy { it.name.values.firstOrNull() }
+    val detectedProjects = detectProjects(lockFile, configFile, platforms)
 
     val currentProjects = lockFile.getAllProjects().filterNot {
         it.slug.keys.firstOrNull() == GitHub.serialName && it.slug.keys.size == 1
     }
 
     val addedProjects = detectedProjects
         .filter { detectedProject -> currentProjects containNotProject detectedProject }
-        .withFoundSubpaths()
         .toSet()
 
     val removedProjects = currentProjects
         .filter { currentProject -> detectedProjects containNotProject currentProject }
-        .withFoundSubpaths()
         .toSet()
 
     val updatedProjects = Multiplatform.platforms.fold(currentProjects) { accProjects, platform ->
@@ -165,7 +44,6 @@ suspend fun syncProjects(
         }
     }
         .distinctBy { it.files }
-        .withFoundSubpaths()
         .filter { project -> project !in currentProjects }
         .toSet()
 

src/commonMain/kotlin/teksturepako/pakku/api/data/Dirs.kt

@@ -6,4 +6,5 @@ object Dirs
 {
     const val PAKKU_DIR = ".pakku"
     val cacheDir = Path(workingPath, "build", ".cache")
+    val shelfDir = Path(workingPath, PAKKU_DIR, "shelf")
 }