From 5280e3c85e75a7d3228cfc9fa976223fd5fa3c73 Mon Sep 17 00:00:00 2001 From: mynttt Date: Thu, 16 Feb 2023 07:23:59 +0100 Subject: [PATCH] 1.7.1 - reinistated TVDB v3 API --- README.md | 1 - VERSION | 2 +- build.gradle | 2 +- src/main/java/updatetool/api/Pipeline.java | 6 +- .../java/updatetool/common/Capabilities.java | 3 +- .../java/updatetool/common/ErrorReports.java | 15 ---- .../java/updatetool/imdb/ImdbPipeline.java | 49 +----------- .../java/updatetool/imdb/ImdbXmlWorker.java | 79 ------------------- src/main/resources/VERSION | 2 +- src/main/resources/desc/imdb-docker.ez | 1 - 10 files changed, 7 insertions(+), 153 deletions(-) delete mode 100644 src/main/java/updatetool/common/ErrorReports.java delete mode 100644 src/main/java/updatetool/imdb/ImdbXmlWorker.java diff --git a/README.md b/README.md index ab69f75..6358015 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,6 @@ Flag | Description :-------------------------:|:-------------------------:| `NO_TV` |Ignore all TV Show libraries `NO_MOVIE` | Ignore all Movie libraries -`VERBOSE_XML_ERROR_LOG` | Enable verbose XML error output logging `DONT_THROW_ON_ENCODING_ERROR` | Supress forced quits if decoding errors of extra data are encountered due to corrupt items in the library `IGNORE_NO_MATCHING_RESOLVER_LOG`|Supresses printing items that have no matching resolver to the log `IGNORE_SCRAPER_NO_RESULT_LOG`|Supresses printing web scraper no-match results that either have no rating on the IMDB website or are not allowed to be rated by anyone on the IMDB website and thus will never have ratings diff --git a/VERSION b/VERSION index 9dbb0c0..081af9a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.0 \ No newline at end of file +1.7.1 \ No newline at end of file diff --git a/build.gradle b/build.gradle index fca1643..40fd666 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { id 'com.github.spotbugs' version '2.0.1' } -version = '1.7.0' +version = '1.7.1' sourceCompatibility = '11' new File(projectDir, "VERSION").text = version; diff --git a/src/main/java/updatetool/api/Pipeline.java b/src/main/java/updatetool/api/Pipeline.java index 01c8dbb..54f26b0 100644 --- a/src/main/java/updatetool/api/Pipeline.java +++ b/src/main/java/updatetool/api/Pipeline.java @@ -3,7 +3,7 @@ public abstract class Pipeline { public enum PipelineStage { - CREATED, ANALYSED_DB, ACCUMULATED_META, TRANSFORMED_META, DB_UPDATED, COMPLETED + CREATED, ANALYSED_DB, ACCUMULATED_META, TRANSFORMED_META, COMPLETED } public final void invoke(T job) throws Exception { @@ -19,9 +19,6 @@ public final void invoke(T job) throws Exception { case CREATED: analyseDatabase(job); break; - case DB_UPDATED: - updateXML(job); - break; case TRANSFORMED_META: updateDatabase(job); break; @@ -34,5 +31,4 @@ public final void invoke(T job) throws Exception { public abstract void accumulateMetadata(T job) throws Exception; public abstract void transformMetadata(T job) throws Exception; public abstract void updateDatabase(T job) throws Exception; - public abstract void updateXML(T job) throws Exception; } \ No newline at end of file diff --git a/src/main/java/updatetool/common/Capabilities.java b/src/main/java/updatetool/common/Capabilities.java index 63356d7..a6dd626 100644 --- a/src/main/java/updatetool/common/Capabilities.java +++ b/src/main/java/updatetool/common/Capabilities.java @@ -9,14 +9,13 @@ public enum Capabilities { TVDB, NO_TV, NO_MOVIE, - VERBOSE_XML_ERROR_LOG, DONT_THROW_ON_ENCODING_ERROR, IGNORE_SCRAPER_NO_RESULT_LOG, IGNORE_NO_MATCHING_RESOLVER_LOG, DISABLE_SCREEN_SCRAPE, PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS; - private static final List USER_FLAGS = List.of(NO_MOVIE, NO_TV, DONT_THROW_ON_ENCODING_ERROR, VERBOSE_XML_ERROR_LOG, IGNORE_NO_MATCHING_RESOLVER_LOG, IGNORE_SCRAPER_NO_RESULT_LOG, DISABLE_SCREEN_SCRAPE, PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS); + private static final List USER_FLAGS = List.of(NO_MOVIE, NO_TV, DONT_THROW_ON_ENCODING_ERROR, IGNORE_NO_MATCHING_RESOLVER_LOG, IGNORE_SCRAPER_NO_RESULT_LOG, DISABLE_SCREEN_SCRAPE, PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS); public static List getUserFlags() { return USER_FLAGS; diff --git a/src/main/java/updatetool/common/ErrorReports.java b/src/main/java/updatetool/common/ErrorReports.java deleted file mode 100644 index 18624e4..0000000 --- a/src/main/java/updatetool/common/ErrorReports.java +++ /dev/null @@ -1,15 +0,0 @@ -package updatetool.common; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.Collection; -import updatetool.Main; - -public class ErrorReports { - - public static void fileReport(Collection nofile, String errorFile) throws IOException { - Files.write(Main.PWD.resolve(errorFile), nofile, StandardCharsets.UTF_8); - } - -} diff --git a/src/main/java/updatetool/imdb/ImdbPipeline.java b/src/main/java/updatetool/imdb/ImdbPipeline.java index 3bbee37..9a9ec30 100644 --- a/src/main/java/updatetool/imdb/ImdbPipeline.java +++ b/src/main/java/updatetool/imdb/ImdbPipeline.java @@ -3,7 +3,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; @@ -12,26 +11,21 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; -import javax.xml.parsers.DocumentBuilderFactory; import org.sqlite.SQLiteException; import org.tinylog.Logger; -import com.google.common.collect.Lists; import updatetool.api.AgentResolvementStrategy; import updatetool.api.ExportedRating; import updatetool.api.Pipeline; import updatetool.common.Capabilities; import updatetool.common.DatabaseSupport.LibraryType; -import updatetool.common.ErrorReports; import updatetool.common.KeyValueStore; import updatetool.common.SqliteDatabaseProvider; import updatetool.common.Utility; @@ -57,7 +51,6 @@ public class ImdbPipeline extends Pipeline { + "|(?agents.thetvdb:\\/\\/)" ); - private static final int LIST_PARTITIONS = 16; private static final int RETRY_N_SECONDS_IF_DB_LOCKED = 20; private static final int ABORT_DB_LOCK_WAITING_AFTER_N_RETRIES = 500; @@ -217,7 +210,7 @@ public void transformMetadata(ImdbJob job) throws Exception { public void updateDatabase(ImdbJob job) throws Exception { if(job.items.isEmpty()) { Logger.info("Nothing to update. Skipping..."); - job.stage = PipelineStage.DB_UPDATED; + job.stage = PipelineStage.COMPLETED; return; } Logger.info("Updating " + job.items.size() + " via batch request..."); @@ -238,48 +231,10 @@ public void updateDatabase(ImdbJob job) throws Exception { } } Logger.info("Batch request finished successfully. Database is now up to date!"); - job.stage = PipelineStage.DB_UPDATED; + job.stage = PipelineStage.COMPLETED; } catch(Exception e) { throw Utility.rethrow(e); } } - @Override - public void updateXML(ImdbJob job) throws Exception { - Logger.info("Updating XML fallback files for " + job.items.size() + " item(s)."); - int n = job.items.size()/LIST_PARTITIONS; - var sublists = Lists.partition(job.items, n == 0 ? 1 : n); - var factory = DocumentBuilderFactory.newInstance(); - AtomicInteger counter = new AtomicInteger(); - HashMap, ImdbXmlWorker> map = new HashMap<>(); - var nofile = Collections.synchronizedCollection(new ArrayList()); - for(var sub : sublists) { - var worker = new ImdbXmlWorker(sub, factory.newDocumentBuilder(), counter, job.items.size(), nofile, configuration.metadataRoot); - map.put(service.submit(worker), worker); - } - Throwable t = null; - List> cleanup = new ArrayList<>(); - for(var entry : map.entrySet()) { - try { - entry.getKey().get(); - } catch(ExecutionException e) { - t = e.getCause(); - } - cleanup.add(entry.getValue().completed); - } - for(var c : cleanup) - job.items.removeAll(c); - if(nofile.size() > 0 && configuration.capabilities.contains(Capabilities.VERBOSE_XML_ERROR_LOG)) { - String errorFile = "xml-error-" + job.uuid + "-" + job.library + ".log"; - Logger.warn(nofile.size() + " XML file(s) have failed to be updated due to them not being present on the file system."); - Logger.warn("This is not an issue as they're not important for Plex as it reads the ratings from the database."); - Logger.warn("The files have been dumped as " + errorFile + " in the PWD."); - ErrorReports.fileReport(nofile, errorFile); - } - if(t != null) - throw Utility.rethrow(t); - Logger.info("Completed updating of XML fallback files."); - job.stage = PipelineStage.COMPLETED; - } - } diff --git a/src/main/java/updatetool/imdb/ImdbXmlWorker.java b/src/main/java/updatetool/imdb/ImdbXmlWorker.java deleted file mode 100644 index fcf7aba..0000000 --- a/src/main/java/updatetool/imdb/ImdbXmlWorker.java +++ /dev/null @@ -1,79 +0,0 @@ -package updatetool.imdb; - -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import org.tinylog.Logger; -import org.w3c.dom.Document; -import updatetool.common.Utility; -import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; - -class ImdbXmlWorker implements Callable { - private final List sub; - private final Collection nofile; - private final DocumentBuilder builder; - private final AtomicInteger counter; - private final Path metadataRoot; - private final int n; - final List completed = new ArrayList<>(); - - ImdbXmlWorker(List sub, DocumentBuilder builder, AtomicInteger counter, int n, Collection nofile, Path metadataRoot) { - this.sub = sub; - this.builder = builder; - this.counter = counter; - this.n = n; - this.nofile = nofile; - this.metadataRoot = metadataRoot; - } - - @Override - public Void call() throws Exception { - for(var item : sub) { - Path contents = metadataRoot.resolve(item.hash.charAt(0)+"/"+item.hash.substring(1)+".bundle/Contents"); - Path imdb = contents.resolve("com.plexapp.agents.imdb/Info.xml"); - Path combined = contents.resolve("_combined/Info.xml"); - try { - transformXML(item, imdb, builder); - transformXML(item, combined, builder); - } catch(Exception e) { - Logger.info("Uncaught exception @ XML Worker: Continuing... ({})", e.getClass().getSimpleName()); - } - int c = counter.incrementAndGet(); - if(c % 100 == 0) - Logger.info("Transforming [{}/{}]...", c, n); - completed.add(item); - } - return null; - } - - private void transformXML(ImdbMetadataResult item, Path p, DocumentBuilder builder) throws Exception { - Document document; - try(var stream = Files.newInputStream(p)) { - document = builder.parse(stream); - } catch (NoSuchFileException e) { - nofile.add(p.toAbsolutePath().toString()); - return; - } - var children = document.getDocumentElement().getChildNodes(); - for(int i = 0; i < children.getLength(); i++) { - if(children.item(i).getNodeName().equals("rating")) - children.item(i).setTextContent(Utility.doubleToOneDecimalString(item.rating)); - if(children.item(i).getNodeName().equals("rating_image")) - children.item(i).setTextContent("imdb://image.rating"); - } - Transformer transformer = TransformerFactory.newInstance().newTransformer(); - try(var stream = Files.newOutputStream(p)) { - transformer.transform(new DOMSource(document), new StreamResult(Files.newOutputStream(p))); - } - } -} diff --git a/src/main/resources/VERSION b/src/main/resources/VERSION index 9dbb0c0..081af9a 100644 --- a/src/main/resources/VERSION +++ b/src/main/resources/VERSION @@ -1 +1 @@ -1.7.0 \ No newline at end of file +1.7.1 \ No newline at end of file diff --git a/src/main/resources/desc/imdb-docker.ez b/src/main/resources/desc/imdb-docker.ez index 5e1206d..180c016 100644 --- a/src/main/resources/desc/imdb-docker.ez +++ b/src/main/resources/desc/imdb-docker.ez @@ -18,7 +18,6 @@ meta { | Currently available: | - NO_TV => Ignore all TV Show libraries | - NO_MOVIE => Ignore all Movie libraries - | - VERBOSE_XML_ERROR_LOG => Enable verbose XML error output logging | - DONT_THROW_ON_ENCODING_ERROR => Supress forced quits if decoding errors of extra data are encountered due to corrupt items in the library | - IGNORE_NO_MATCHING_RESOLVER_LOG => Supresses printing items that have no matching resolver to the log | - IGNORE_SCRAPER_NO_RESULT_LOG => Supresses printing web scraper no-match results that either have no rating on the IMDB website or are not allowed to be rated by anyone on the IMDB website and thus will never have ratings