feat: implement streaming database import and update API integration

- Removed the obsolete db.jsonl file and replaced it with a dynamic API-based import process. - Enhanced AnimeImporter to support streaming data reading from a URL, improving efficiency. - Introduced StreamReader for handling large data files line-by-line, optimizing memory usage. - Updated IchimeApp to fetch the latest database version from the API before importing. - Improved error handling and progress tracking during the import process.
midori-no-me · Dec 16, 2024 · d2d2299 · d2d2299
1 parent abd288c
commit d2d2299
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 156 deletions.
diff --git a/Ichime/DependencyInjection/DI.swift b/Ichime/DependencyInjection/DI.swift
@@ -13,41 +13,6 @@ import ShikimoriApiClient
 import SwiftData
 
 class ApplicationDependency: DIFramework {
-  private static func setupInitialDatabase() {
-    guard
-      let groupContainerURL = FileManager.default.containerURL(
-        forSecurityApplicationGroupIdentifier: ServiceLocator.appGroup
-      )
-    else {
-      print("Не удалось получить URL группового контейнера")
-      return
-    }
-
-    let cachesDirURL = groupContainerURL.appendingPathComponent("Library/Caches")
-    let defaultStoreURL = cachesDirURL.appendingPathComponent("default.store")
-
-    // Проверяем существует ли файл
-    if !FileManager.default.fileExists(atPath: defaultStoreURL.path) {
-      do {
-        // Создаем директорию Caches если её нет
-        try FileManager.default.createDirectory(at: cachesDirURL, withIntermediateDirectories: true)
-
-        // Получаем URL исходного файла из бандла приложения
-        guard let sourceURL = Bundle.main.url(forResource: "default", withExtension: "store") else {
-          print("Не найден исходный файл default.store в бандле")
-          return
-        }
-
-        // Копируем файл
-        try FileManager.default.copyItem(at: sourceURL, to: defaultStoreURL)
-        UserDefaults().set(true, forKey: "firstLaunch")
-        print("База данных успешно скопирована в: \(defaultStoreURL.path)")
-      }
-      catch {
-        print("Ошибка при копировании базы данных: \(error)")
-      }
-    }
-  }
 
   static let container: DIContainer = {
     let container = DIContainer()
@@ -56,9 +21,6 @@ class ApplicationDependency: DIFramework {
   }()
 
   static func load(container: DIContainer) {
-    // Инициализируем базу данных перед созданием ModelContainer
-    //    setupInitialDatabase()
-
     container.register {
       let schema = Schema([
         UserAnimeListModel.self,

diff --git a/Ichime/IchimeApp.swift b/Ichime/IchimeApp.swift
@@ -31,15 +31,6 @@ struct IchimeApp: App {
         .onAppear {
           VideoPlayerController.enableBackgroundMode()
           NotificationCounterWatcher.askBadgePermission()
-
-          if let containerURL = FileManager.default.containerURL(
-            forSecurityApplicationGroupIdentifier: ServiceLocator.appGroup
-          ) {
-            print("Group Container Path: \(containerURL.path)")
-          }
-          else {
-            print("Failed to get group container path")
-          }
         }
         .task {
           await importDatabase()
@@ -60,36 +51,51 @@ struct IchimeApp: App {
     guard !isImporting else { return }
     isImporting = true
 
-    let animeImporter = AnimeImporter(modelContainer: container)
+    do {
+      // Запрос к API
+      let url = URL(string: "https://db.dimensi.dev/api/latest")!
+      let (data, _) = try await URLSession.shared.data(from: url)
+      let response = try JSONDecoder().decode(DbServerResponse.self, from: data)
 
-    let lastUpdated = UserDefaults().string(forKey: "lastUpdated") ?? ""
-    let lastTimestamp = Int(lastUpdated) ?? 0
-    let timestamp = await animeImporter.getTimestamp()
+      let lastUpdated = UserDefaults().string(forKey: "lastUpdated") ?? ""
+      let lastTimestamp = Int(lastUpdated) ?? 0
 
-    if lastTimestamp == timestamp {
-      isImporting = false
-      return
-    }
+      if lastTimestamp == response.date {
+        isImporting = false
+        return
+      }
 
-    Task.detached(priority: .high) {
-      do {
-        async let result: () = animeImporter.importDatabase(from: "db.jsonl")
-        for await progress in await animeImporter.currentProgress {
+      let animeImporter = AnimeImporter(modelContainer: container)
+      Task.detached(priority: .high) {
+        do {
+          let dbUrl = "https://db.dimensi.dev\(response.url)"
+          async let result: () = animeImporter.importDatabase(from: dbUrl)
+          for await progress in await animeImporter.currentProgress {
+            await MainActor.run {
+              progressText = progress
+            }
+          }
+          try await result
           await MainActor.run {
-            progressText = progress
+            isImporting = false
+            UserDefaults().set(response.date, forKey: "lastUpdated")
           }
         }
-        try await result
-        await MainActor.run {
-          isImporting = false
-          UserDefaults().set(timestamp, forKey: "lastUpdated")
-        }
-      }
-      catch {
-        await MainActor.run {
-          isImporting = false
+        catch {
+          await MainActor.run {
+            isImporting = false
+          }
         }
       }
     }
+    catch {
+      isImporting = false
+      print("Error fetching database info: \(error)")
+    }
   }
 }
+
+struct DbServerResponse: Codable {
+  let date: Int
+  let url: String
+}
diff --git a/Ichime/OfflineDb/AnimeImporter.swift b/Ichime/OfflineDb/AnimeImporter.swift
@@ -29,33 +29,29 @@ actor AnimeImporter {
     return stream
   }
 
-  func getTimestamp() -> Int {
-    let reader = DbReader(fileName: "db.jsonl")
-    guard let reader = reader else {
-      print("Failed to initialize DB reader")
-      return 0
-    }
-
-    defer {
-      reader.close()
-    }
-    return reader.getTimestamp()
-  }
-
   private var genresMap: [Int: DbGenre] = [:]
   private var studioMap: [Int: DbStudio] = [:]
   private var charactersMap: [Int: DbCharacter] = [:]
   private let batchSize = 250
 
-  func importDatabase(from path: String) async throws {
-    let reader = DbReader(fileName: path)
-    guard let reader = reader else {
-      print("Failed to initialize DB reader")
+  func importDatabase(from url: String) async throws {
+    progressContinuation?.yield("Загрузка базы данных...")
+
+    guard let url = URL(string: url) else {
+      print("Invalid URL")
+      return
+    }
+
+    let (fileUrl, _) = try await URLSession.shared.download(from: url)
+
+    guard let reader = StreamReader(path: fileUrl.path) else {
+      print("Invalid file")
       return
     }
 
     defer {
-      // Завершаем стрим
+      reader.close()
+      try? FileManager.default.removeItem(at: fileUrl)
       progressContinuation?.finish()
       genresMap = [:]
       studioMap = [:]
@@ -64,26 +60,28 @@ actor AnimeImporter {
       }
     }
 
-    print("Importing database... \(reader.getTimestamp())")
-
     modelContext.autosaveEnabled = false
-
     let startTime = Date()
 
     let decoder = JSONDecoder()
     decoder.dateDecodingStrategy = .iso8601
 
     var models: [DbAnime] = []
-
     let startParseTime = Date()
 
-    progressContinuation?.yield("Парсим данных из источника")
+    progressContinuation?.yield("Парсим данные из источника")
 
     let cachedDb = try modelContext.fetch(FetchDescriptor<DbAnime>()).map { $0.id }
 
-    for jsonData in reader {
+    // Читаем файл построчно
+    while let line = reader.nextLine() {
+      guard !line.isEmpty else {
+        fatalError("Empty line")
+      }
+
       do {
-        let jsonAnime = try decoder.decode(JsonAnime.self, from: jsonData)
+        guard let data = line.data(using: .utf8) else { continue }
+        let jsonAnime = try decoder.decode(JsonAnime.self, from: data)
         if cachedDb.contains(jsonAnime.id) {
           continue
         }
@@ -95,8 +93,6 @@ actor AnimeImporter {
       }
     }
 
-    reader.close()
-
     print("Parsed \(models.count) in \(Date().timeIntervalSince(startParseTime)) seconds")
 
     progressContinuation?.yield("Сохраняем данные в нашей базе")

diff --git a/Ichime/OfflineDb/DbReader.swift → Ichime/OfflineDb/StreamReader.swift b/Ichime/OfflineDb/DbReader.swift → Ichime/OfflineDb/StreamReader.swift
@@ -1,5 +1,5 @@
 //
-//  DbReader.swift
+//  StreamReader.swift
 //  Ichime
 //
 //  Created by Nafranets Nikita on 13.12.2024.
@@ -8,62 +8,6 @@
 import CoreData
 import Foundation
 
-class DbReader {
-  private let streamReader: StreamReader
-
-  init?(fileName: String) {
-    guard let filePath = Bundle.main.path(forResource: fileName, ofType: nil),
-      let reader = StreamReader(path: filePath)
-    else {
-      return nil
-    }
-
-    self.streamReader = reader
-  }
-
-  func close() {
-    streamReader.close()
-  }
-
-  func rewind() {
-    streamReader.rewind()
-    let _ = nextLine()
-  }
-
-  func getTimestamp() -> Int {
-    let decoder = JSONDecoder()
-    let line = nextLine()
-    guard let line = line else {
-      return 0
-    }
-    guard let timestamp = try? decoder.decode(JsonTimestamp.self, from: line) else {
-      return 0
-    }
-
-    return timestamp.timestamp
-  }
-
-  func nextLine() -> Data? {
-    guard let line = streamReader.nextLine() else {
-      return nil
-    }
-
-    return line.data(using: .utf8)
-  }
-}
-
-extension DbReader: Sequence {
-  func makeIterator() -> AnyIterator<Data> {
-    AnyIterator {
-      self.nextLine()
-    }
-  }
-}
-
-struct JsonTimestamp: Codable {
-  let timestamp: Int
-}
-
 class StreamReader {
 
   let encoding: String.Encoding

diff --git a/Ichime/db.jsonl b/Ichime/db.jsonl