From 9adaa3d818d234577f26023d76e549f1b96edeb6 Mon Sep 17 00:00:00 2001 From: mynttt Date: Thu, 16 Feb 2023 07:18:36 +0100 Subject: [PATCH 1/2] Revert "1.7.0 (2) - cleaned up code base + TVDB API optimization" This reverts commit 6580dceaf186e02d5331a822bd44bb495df76ad7. --- CHANGELOG.md | 3 + README.md | 10 +- src/main/java/updatetool/api/Pipeline.java | 6 +- .../java/updatetool/common/Capabilities.java | 5 +- .../java/updatetool/common/ErrorReports.java | 15 + .../common/externalapis/AbstractApi.java | 2 +- .../common/externalapis/TmdbApiV4.java | 22 ++ .../common/externalapis/TvdbApiV3.java | 279 ++++++++++++++++++ .../common/externalapis/TvdbApiV4.java | 18 +- .../imdb/ImdbDockerImplementation.java | 22 +- .../java/updatetool/imdb/ImdbPipeline.java | 61 +++- .../java/updatetool/imdb/ImdbXmlWorker.java | 79 +++++ .../NewPlexAgentToImdbResolvement.java | 4 + src/main/resources/desc/imdb-docker.ez | 1 + 14 files changed, 505 insertions(+), 22 deletions(-) create mode 100644 src/main/java/updatetool/common/ErrorReports.java create mode 100644 src/main/java/updatetool/common/externalapis/TmdbApiV4.java create mode 100644 src/main/java/updatetool/common/externalapis/TvdbApiV3.java create mode 100644 src/main/java/updatetool/imdb/ImdbXmlWorker.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b69aafe..ad6b563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 1.7.1 +- reinistated TVDB v3 API as it is still functional + ## 1.7.0 - Removed transactions from native SQLite binary interaction + using batch mode on binary to hopefully mitigate further SQLite corruption issues - Added `PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS` capability to further diagnose future potential SQLite statement/corruption issues diff --git a/README.md b/README.md index d78a217..ab69f75 100644 --- a/README.md +++ b/README.md @@ -71,12 +71,19 @@ Name | Description `USE_PLEX_SQLITE_BINARY_FOR_WRITE_ACCESS`|Allows to use the non-standard Plex SQLite3 version that diverged so strongly from the vanilla flavour that write operations with vanilla SQLite3 can cause database corruptions! Set this to `true` in the docker when using a version >= 1.6.0! This is the only way to be safe from corruptions as Plex continues to diverge from compatibility with vanilla SQLite3! If you're not using a docker version make sure that this points to the `Plex Media Server/Plex SQLite` binary that is located in the main Plex folder next to the `Plex Media Server` executable. `OVERRIDE_DATABASE_LOCATION`|Overrides the path where UpdateTool looks for the Plex database. The database needs to be contained in this folder. Useful if a docker container uses a volume and a different path structure. ([more here](#override-the-database-location)) `TMDB_API_KEY`|Enables TMDB Movie/Series library processing -`TVDB_API_KEY`|Enables TVDB Series library processing using the v4 pin +`TVDB_API_KEY`|Enables TVDB Series library processing using either the v3 legacy key or the v4 pin `UNLOCK_FOR_NEW_TV_AGENT`|Opt-in for libraries using the new TV Show agent. All libraries that are opted-in this way will have their ratings changed to IMDB ratings by this tool ([more here](#opt-in-for-libraries-using-the-new-tv-show-agent)) `IGNORE_LIBS`|Ignore libraries with certain IDs ([more here](#Ignore-libraries-from-being-updated)) `CAPABILITIES`|Custom flags for the tool ([more here](#supply-custom-capability-flags)) `JVM_MAX_HEAP`|Only relevant for the docker. Specify max. heap allocatable by the JVM (default 256m). Can be useful if you have a really large library (40000+ items) and you run in memory related crashes. Must be specified in bytes (i.e. 256m, 1g, 2g, 512m) +Deprecated variables can still be used although their usage is discouraged. + +### Deprecated Environment Variables +Name | Description | Deprecation +:-------------------------:|:-------------------------:|:-------------------------:| +`TVDB_AUTH_STRING`|Enables TVDB Series library processing|API Key is enough for this tool to work + ## Docker on UnRaid There is a template repository available now: https://github.com/mynttt/unraid-templates @@ -210,6 +217,7 @@ Flag | Description :-------------------------:|:-------------------------:| `NO_TV` |Ignore all TV Show libraries `NO_MOVIE` | Ignore all Movie libraries +`VERBOSE_XML_ERROR_LOG` | Enable verbose XML error output logging `DONT_THROW_ON_ENCODING_ERROR` | Supress forced quits if decoding errors of extra data are encountered due to corrupt items in the library `IGNORE_NO_MATCHING_RESOLVER_LOG`|Supresses printing items that have no matching resolver to the log `IGNORE_SCRAPER_NO_RESULT_LOG`|Supresses printing web scraper no-match results that either have no rating on the IMDB website or are not allowed to be rated by anyone on the IMDB website and thus will never have ratings diff --git a/src/main/java/updatetool/api/Pipeline.java b/src/main/java/updatetool/api/Pipeline.java index 54f26b0..01c8dbb 100644 --- a/src/main/java/updatetool/api/Pipeline.java +++ b/src/main/java/updatetool/api/Pipeline.java @@ -3,7 +3,7 @@ public abstract class Pipeline { public enum PipelineStage { - CREATED, ANALYSED_DB, ACCUMULATED_META, TRANSFORMED_META, COMPLETED + CREATED, ANALYSED_DB, ACCUMULATED_META, TRANSFORMED_META, DB_UPDATED, COMPLETED } public final void invoke(T job) throws Exception { @@ -19,6 +19,9 @@ public final void invoke(T job) throws Exception { case CREATED: analyseDatabase(job); break; + case DB_UPDATED: + updateXML(job); + break; case TRANSFORMED_META: updateDatabase(job); break; @@ -31,4 +34,5 @@ public final void invoke(T job) throws Exception { public abstract void accumulateMetadata(T job) throws Exception; public abstract void transformMetadata(T job) throws Exception; public abstract void updateDatabase(T job) throws Exception; + public abstract void updateXML(T job) throws Exception; } \ No newline at end of file diff --git a/src/main/java/updatetool/common/Capabilities.java b/src/main/java/updatetool/common/Capabilities.java index 32bed63..63356d7 100644 --- a/src/main/java/updatetool/common/Capabilities.java +++ b/src/main/java/updatetool/common/Capabilities.java @@ -8,14 +8,15 @@ public enum Capabilities { TMDB, TVDB, NO_TV, - NO_MOVIE, + NO_MOVIE, + VERBOSE_XML_ERROR_LOG, DONT_THROW_ON_ENCODING_ERROR, IGNORE_SCRAPER_NO_RESULT_LOG, IGNORE_NO_MATCHING_RESOLVER_LOG, DISABLE_SCREEN_SCRAPE, PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS; - private static final List USER_FLAGS = List.of(NO_MOVIE, NO_TV, DONT_THROW_ON_ENCODING_ERROR, IGNORE_NO_MATCHING_RESOLVER_LOG, IGNORE_SCRAPER_NO_RESULT_LOG, DISABLE_SCREEN_SCRAPE, PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS); + private static final List USER_FLAGS = List.of(NO_MOVIE, NO_TV, DONT_THROW_ON_ENCODING_ERROR, VERBOSE_XML_ERROR_LOG, IGNORE_NO_MATCHING_RESOLVER_LOG, IGNORE_SCRAPER_NO_RESULT_LOG, DISABLE_SCREEN_SCRAPE, PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS); public static List getUserFlags() { return USER_FLAGS; diff --git a/src/main/java/updatetool/common/ErrorReports.java b/src/main/java/updatetool/common/ErrorReports.java new file mode 100644 index 0000000..18624e4 --- /dev/null +++ b/src/main/java/updatetool/common/ErrorReports.java @@ -0,0 +1,15 @@ +package updatetool.common; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Collection; +import updatetool.Main; + +public class ErrorReports { + + public static void fileReport(Collection nofile, String errorFile) throws IOException { + Files.write(Main.PWD.resolve(errorFile), nofile, StandardCharsets.UTF_8); + } + +} diff --git a/src/main/java/updatetool/common/externalapis/AbstractApi.java b/src/main/java/updatetool/common/externalapis/AbstractApi.java index 15ff44b..cb4f30f 100644 --- a/src/main/java/updatetool/common/externalapis/AbstractApi.java +++ b/src/main/java/updatetool/common/externalapis/AbstractApi.java @@ -16,7 +16,7 @@ public abstract class AbstractApi { public enum ApiVersion { - TMDB_V3, TMDB_V4, TVDB_V4; + TMDB_V3, TMDB_V4, TVDB_V3, TVDB_V4; } private final HttpClient client; diff --git a/src/main/java/updatetool/common/externalapis/TmdbApiV4.java b/src/main/java/updatetool/common/externalapis/TmdbApiV4.java new file mode 100644 index 0000000..3ff5944 --- /dev/null +++ b/src/main/java/updatetool/common/externalapis/TmdbApiV4.java @@ -0,0 +1,22 @@ +package updatetool.common.externalapis; + +import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; + +//TODO: new v4 support and v3 legacy lookup +// TMDB API v4 appears to only be used to manage the user account and user account items like personal watchlists (07.02.2022) + +public class TmdbApiV4 extends AbstractApi implements TmdbApi { + + @Override + public void resolveImdbIdForItem(ImdbMetadataResult result) { + // TODO Auto-generated method stub + + } + + @Override + public ApiVersion version() { + return ApiVersion.TMDB_V3; + } + + +} diff --git a/src/main/java/updatetool/common/externalapis/TvdbApiV3.java b/src/main/java/updatetool/common/externalapis/TvdbApiV3.java new file mode 100644 index 0000000..fd4caf5 --- /dev/null +++ b/src/main/java/updatetool/common/externalapis/TvdbApiV3.java @@ -0,0 +1,279 @@ +package updatetool.common.externalapis; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.Map; +import java.util.Objects; +import org.tinylog.Logger; +import com.google.gson.Gson; +import com.google.gson.internal.LinkedTreeMap; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.ParseContext; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import net.minidev.json.JSONArray; +import updatetool.common.DatabaseSupport.LibraryType; +import updatetool.common.HttpRunner; +import updatetool.common.KeyValueStore; +import updatetool.common.Utility; +import updatetool.common.HttpRunner.Converter; +import updatetool.common.HttpRunner.Handler; +import updatetool.common.HttpRunner.HttpCodeHandler; +import updatetool.common.HttpRunner.RunnerResult; +import updatetool.exceptions.ApiCallFailedException; +import updatetool.imdb.ImdbUtility; +import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; + +public class TvdbApiV3 extends AbstractApi implements TvdbApi { + private static final ParseContext CTX = JsonPath.using(Configuration.defaultConfiguration().setOptions(Option.SUPPRESS_EXCEPTIONS)); + private static final String BASE_URL = "https://api.thetvdb.com"; + private String authToken; + private final Gson gson = new Gson(); + private final KeyValueStore cache, blacklist, cacheMovie, blacklistMovie; + private final HttpRunner runner; + private final HttpRunner runnerMovie; + + private class UnmarshalTvdb { + public final String Error = null; + private final Object data = null; + + private boolean isSeries() { + return data instanceof LinkedTreeMap; + } + + private boolean isEpisode() { + return data instanceof ArrayList; + } + + @SuppressWarnings("rawtypes") + private String getImdbId() { + if(data == null) return null; + if(isSeries()) return (String) ((LinkedTreeMap) data).get("imdbId"); + if(isEpisode()) return (String) ((LinkedTreeMap) ((ArrayList) data).get(0)).get("imdbId"); + return null; + } + } + + private class Token { String token; }; + + @SuppressFBWarnings("DM_EXIT") + public TvdbApiV3(String key, KeyValueStore cache, KeyValueStore blacklist, KeyValueStore cacheMovie, KeyValueStore blacklistMovie) throws ApiCallFailedException { + Logger.info("Testing TVDB API (v3) authorization apikey: {}", key); + + try { + authToken = "Bearer " + auth(key); + } catch(ApiCallFailedException e) { + Logger.error("API Test failed: " + e.getMessage()); + Logger.error("Legacy (v3) keys available under: https://thetvdb.com/"); + System.exit(-1); + } + + Logger.info("Test passed. API Key is valid."); + this.cache = cache; + this.blacklist = blacklist; + this.blacklistMovie = blacklistMovie; + this.cacheMovie = cacheMovie; + + Converter converter = resp -> { + return Objects.requireNonNull(gson.fromJson(resp.body(), UnmarshalTvdb.class)); + }; + + Handler handler = (resp, res, payload) -> { + if(res.Error != null) { + Logger.error("TVDB item (v3) {} with id {} reported error: {}", payload.title, payload.extractedId, res.Error); + blacklist.cache(payload.extractedId, ""); + return RunnerResult.ofSuccess(res); + } + + String imdbId = res.getImdbId(); + + if(imdbId != null && !imdbId.isBlank()) { + cache.cache(payload.extractedId, imdbId); + payload.imdbId = imdbId; + payload.resolved = true; + } else { + blacklist.cache(payload.extractedId, ""); + Logger.warn("TVDB item (v3) {} with id {} does not have an IMDB id associated.", payload.title, payload.extractedId); + } + + return RunnerResult.ofSuccess(res); + }; + + Handler handler404 = (resp, res, payload) -> { + blacklist.cache(payload.extractedId, ""); + return RunnerResult.ofSuccess(res); + }; + + Converter converterMovie = resp -> null; + + Handler handlerMovie = (resp, res, payload) -> { + var doc = CTX.parse(resp.body()); + String error = doc.read("$.Error"); + if(error != null) { + Logger.error("TVDB movie item (v3) {} with id {} reported error: {}", payload.title, payload.extractedId, error); + blacklistMovie.cache(payload.extractedId, ""); + return RunnerResult.ofSuccess(res); + } + + String imdbId; + try { + imdbId = (String) ((JSONArray) doc.read("$..remoteids[?(@.source_id == 2)].id")).get(0); + } catch(Exception e) { + return RunnerResult.ofSuccess(res); + } + + if(imdbId != null && !imdbId.isBlank()) { + cacheMovie.cache(payload.extractedId, imdbId); + payload.imdbId = imdbId; + payload.resolved = true; + } else { + blacklistMovie.cache(payload.extractedId, ""); + Logger.warn("TVDB movie item (v3) {} with id {} does not have an IMDB id associated.", payload.title, payload.extractedId); + } + + return RunnerResult.ofSuccess(res); + }; + + Handler handler404m = (resp, res, payload) -> { + blacklistMovie.cache(payload.extractedId, ""); + return RunnerResult.ofSuccess(res); + }; + + this.runner = new HttpRunner<>(converter, HttpCodeHandler.of(Map.of(200, handler, 404, handler404)) ,"TVDB API v3", 3); + this.runnerMovie = new HttpRunner<>(converterMovie, HttpCodeHandler.of(Map.of(200, handlerMovie, 404, handler404m)), "TVDB API v3 (Movie)", 3); + } + + private String auth(String key) throws ApiCallFailedException { + try { + var response = send( + postJson(BASE_URL + "/login", gson.toJson(Map.of( + "apikey", key) + )) + ); + if(response.statusCode() != 200) { + Logger.error("TVDB authorization failed with code {}", response.statusCode()); + Logger.error("This could be due to the TVDB API having issues at the moment or your credentials being wrong."); + Logger.error("This is the received response:"); + Logger.error(response.body()); + Logger.error("==================================================="); + throw new ApiCallFailedException("TVDB API authorization failed."); + } + return new Gson().fromJson(response.body(), Token.class).token; + } catch (IOException | InterruptedException e) { + throw Utility.rethrow(e); + } + } + + private HttpResponse movieImdbId(String tvdbId) { + try { + return send(HttpRequest.newBuilder(new URI(String.format("%s/movies/%s", BASE_URL, tvdbId))) + .GET() + .header("Authorization", authToken) + .build()); + } catch (IOException | InterruptedException | URISyntaxException e) { + throw Utility.rethrow(e); + } + } + + private HttpResponse seriesImdbId(String tvdbId) { + try { + return send(HttpRequest.newBuilder(new URI(String.format("%s/series/%s", BASE_URL, tvdbId))) + .GET() + .header("Authorization", authToken) + .build()); + } catch (IOException | InterruptedException | URISyntaxException e) { + throw Utility.rethrow(e); + } + } + + private HttpResponse episodeImdbId(String[] parts) { + try { + return send(HttpRequest.newBuilder(new URI(String.format("%s/series/%s/episodes/query?airedSeason=%s&airedEpisode=%s", BASE_URL, parts[0], parts[1], parts[2]))) + .GET() + .header("Authorization", authToken) + .build()); + } catch (IOException | InterruptedException | URISyntaxException e) { + throw Utility.rethrow(e); + } + } + + private int categorize(String tvdbId) { + if(ImdbUtility.TVDB_TMDB_EPISODE.matcher(tvdbId).find()) + return 2; + if(ImdbUtility.TVDB_TMDB_SEASON.matcher(tvdbId).find()) + return 1; + if(ImdbUtility.TVDB_TMDB_SERIES.matcher(tvdbId).find()) + return 0; + throw new IllegalArgumentException("This should never happen! Input was: " + tvdbId); + } + + @Override + public void resolveImdbIdForItem(ImdbMetadataResult result) { + if(result.type == LibraryType.MOVIE) { + result.extractedId = ImdbUtility.extractId(ImdbUtility.TVDB_TMDB_SERIES, result.guid); + + if(result.extractedId == null) { + Logger.error("Item: {} is detected as TVDB Movie (v3) but has no id. (guid={})", result.title, result.guid); + return; + } + + var lookup = cacheMovie.lookup(result.extractedId); + if(lookup != null) { + result.imdbId = lookup; + result.resolved = true; + return; + } + + if(blacklistMovie.lookup(result.extractedId) != null) { + return; + } + + runnerMovie.run(() -> movieImdbId(result.extractedId), result); + } else { + result.extractedId = ImdbUtility.extractId(ImdbUtility.TVDB_TMDB_SERIES_MATCHING, result.guid); + + if(result.extractedId == null) { + Logger.error("Item: {} is detected as TVDB (v3) but has no id. (guid={})", result.title, result.guid); + return; + } + + var lookup = cache.lookup(result.extractedId); + if(lookup != null) { + result.imdbId = lookup; + result.resolved = true; + return; + } + + if(blacklist.lookup(result.extractedId) != null) { + return; + } + + String[] parts = result.extractedId.split("/"); + + switch(categorize(result.extractedId)) { + case 0: + runner.run(() -> seriesImdbId(parts[0]), result); + break; + case 1: + // Seasons: If ever added to IMDB could be implemented here + result.resolved = false; + return; + case 2: + runner.run(() -> episodeImdbId(parts), result); + break; + default: + throw new UnsupportedOperationException(); + } + } + } + + @Override + public ApiVersion version() { + return ApiVersion.TVDB_V3; + } +} diff --git a/src/main/java/updatetool/common/externalapis/TvdbApiV4.java b/src/main/java/updatetool/common/externalapis/TvdbApiV4.java index 7ff34eb..f7db3ac 100644 --- a/src/main/java/updatetool/common/externalapis/TvdbApiV4.java +++ b/src/main/java/updatetool/common/externalapis/TvdbApiV4.java @@ -186,9 +186,9 @@ private String auth(String pin) { } } - private HttpResponse queryForMovie(String id) { + private HttpResponse queryForEpisode(String id) { try { - return send(HttpRequest.newBuilder(new URI(String.format("%s/movies/%s/extended", BASE_URL, id))) + return send(HttpRequest.newBuilder(new URI(String.format("%s/episodes/%s/extended", BASE_URL, id))) .GET() .header("Authorization", authToken) .build()); @@ -199,7 +199,7 @@ private HttpResponse queryForMovie(String id) { private HttpResponse queryForSeries(String id) { try { - return send(HttpRequest.newBuilder(new URI(String.format("%s/series/%s/extended?short=true", BASE_URL, id))) + return send(HttpRequest.newBuilder(new URI(String.format("%s/series/%s/extended", BASE_URL, id))) .GET() .header("Authorization", authToken) .build()); @@ -208,26 +208,26 @@ private HttpResponse queryForSeries(String id) { } } - private HttpResponse queryForSeasons(String id) { + private HttpResponse queryForMovie(String id) { try { - return send(HttpRequest.newBuilder(new URI(String.format("%s/seasons/%s/extended", BASE_URL, id))) + return send(HttpRequest.newBuilder(new URI(String.format("%s/movies/%s/extended", BASE_URL, id))) .GET() .header("Authorization", authToken) .build()); } catch (IOException | InterruptedException | URISyntaxException e) { throw Utility.rethrow(e); - } + } } - private HttpResponse queryForEpisode(String id) { + private HttpResponse queryForSeasons(String id) { try { - return send(HttpRequest.newBuilder(new URI(String.format("%s/episodes/%s/extended", BASE_URL, id))) + return send(HttpRequest.newBuilder(new URI(String.format("%s/seasons/%s/extended", BASE_URL, id))) .GET() .header("Authorization", authToken) .build()); } catch (IOException | InterruptedException | URISyntaxException e) { throw Utility.rethrow(e); - } + } } @Override diff --git a/src/main/java/updatetool/imdb/ImdbDockerImplementation.java b/src/main/java/updatetool/imdb/ImdbDockerImplementation.java index 10c7853..4f76afa 100644 --- a/src/main/java/updatetool/imdb/ImdbDockerImplementation.java +++ b/src/main/java/updatetool/imdb/ImdbDockerImplementation.java @@ -28,6 +28,7 @@ import updatetool.common.Utility; import updatetool.common.externalapis.AbstractApi.ApiVersion; import updatetool.common.externalapis.TmdbApiV3; +import updatetool.common.externalapis.TvdbApiV3; import updatetool.common.externalapis.TvdbApiV4; import updatetool.exceptions.ApiCallFailedException; import updatetool.exceptions.ImdbDatasetAcquireException; @@ -55,6 +56,7 @@ public ImdbDockerImplementation(String id, String desc, String usage, String hel @SuppressFBWarnings("DM_EXIT") public void bootstrap(Map args) throws Exception { apikeyTmdb = System.getenv("TMDB_API_KEY"); + String tvdbAuthLegacy = System.getenv("TVDB_AUTH_STRING"); String tvdbApiKey = System.getenv("TVDB_API_KEY"); String data = System.getenv("PLEX_DATA_DIR"); String ignore = System.getenv("IGNORE_LIBS"); @@ -107,12 +109,28 @@ public void bootstrap(Map args) throws Exception { Logger.info("TMDB API key enabled TMDB <=> IMDB matching. Will process TMDB backed Movie and TV Series libraries and TMDB orphans."); } + if(tvdbAuthLegacy != null && !tvdbAuthLegacy.isBlank()) { + Logger.warn("Don't use legacy environment variable TVDB_AUTH_STRING. Use TVDB_API_KEY instead by only providing the TVDB API key."); + String[] info = tvdbAuthLegacy.split(";"); + if(info.length == 3) { + tvdbApiKey = info[2]; + } else { + Logger.error("Invalid TVDB API authorization string given. Must contain 3 items seperated by a ';'. Will ignore TV Series with the TVDB agent."); + } + } + if(tvdbApiKey == null || tvdbApiKey.isBlank()) { Logger.info("No TVDB API authorization string detected. Will not process TVDB backed Movie and TV Series libraries."); capabilities.remove(Capabilities.TVDB); } else { - ApiVersion version = new TvdbApiV4(tvdbApiKey, null, null, null, null, null).version(); - apiauthTvdb = tvdbApiKey.trim(); + ApiVersion version; + if(tvdbApiKey.length() == 16 || tvdbApiKey.length() >= 32) { + tvdbApiKey = tvdbApiKey.trim(); + version = new TvdbApiV3(tvdbApiKey, null, null, null, null).version(); + } else { + version = new TvdbApiV4(tvdbApiKey, null, null, null, null, null).version(); + } + apiauthTvdb = tvdbApiKey; Logger.info("TVDB API ({}) authorization enabled IMDB rating update for Movies and TV Series with the TVDB agent.", version); } diff --git a/src/main/java/updatetool/imdb/ImdbPipeline.java b/src/main/java/updatetool/imdb/ImdbPipeline.java index 9ae2970..3bbee37 100644 --- a/src/main/java/updatetool/imdb/ImdbPipeline.java +++ b/src/main/java/updatetool/imdb/ImdbPipeline.java @@ -3,6 +3,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; @@ -11,25 +12,31 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; +import javax.xml.parsers.DocumentBuilderFactory; import org.sqlite.SQLiteException; import org.tinylog.Logger; +import com.google.common.collect.Lists; import updatetool.api.AgentResolvementStrategy; import updatetool.api.ExportedRating; import updatetool.api.Pipeline; import updatetool.common.Capabilities; import updatetool.common.DatabaseSupport.LibraryType; +import updatetool.common.ErrorReports; import updatetool.common.KeyValueStore; import updatetool.common.SqliteDatabaseProvider; import updatetool.common.Utility; import updatetool.common.externalapis.TmdbApiV3; +import updatetool.common.externalapis.TvdbApiV3; import updatetool.common.externalapis.TvdbApiV4; import updatetool.exceptions.ApiCallFailedException; import updatetool.exceptions.DatabaseLockedException; @@ -50,6 +57,7 @@ public class ImdbPipeline extends Pipeline { + "|(?agents.thetvdb:\\/\\/)" ); + private static final int LIST_PARTITIONS = 16; private static final int RETRY_N_SECONDS_IF_DB_LOCKED = 20; private static final int ABORT_DB_LOCK_WAITING_AFTER_N_RETRIES = 500; @@ -69,6 +77,7 @@ public static class ImdbPipelineConfiguration { private final EnumSet capabilities; public final String tmdbApiKey, tvdbApiKey, dbLocation, executeUpdatesOverPlexSqliteVersion; public final Path metadataRoot; + public final boolean isTvdbV4; public ImdbPipelineConfiguration(String tmdbApiKey, String tvdbApiKey, Path metadataRoot, String dbLocation, String executeUpdatesOverPlexSqliteVersion, EnumSet capabilities) { this.tmdbApiKey = tmdbApiKey; @@ -77,6 +86,7 @@ public ImdbPipelineConfiguration(String tmdbApiKey, String tvdbApiKey, Path meta this.dbLocation = dbLocation; this.capabilities = capabilities; this.executeUpdatesOverPlexSqliteVersion = executeUpdatesOverPlexSqliteVersion; + this.isTvdbV4 = resolveTvdb() ? !(tvdbApiKey.length() == 16 || tvdbApiKey.length() >= 32) : false; } public boolean resolveTmdb() { @@ -100,10 +110,11 @@ public ImdbPipeline(ImdbLibraryMetadata metadata, ExecutorService service, Map, ImdbXmlWorker> map = new HashMap<>(); + var nofile = Collections.synchronizedCollection(new ArrayList()); + for(var sub : sublists) { + var worker = new ImdbXmlWorker(sub, factory.newDocumentBuilder(), counter, job.items.size(), nofile, configuration.metadataRoot); + map.put(service.submit(worker), worker); + } + Throwable t = null; + List> cleanup = new ArrayList<>(); + for(var entry : map.entrySet()) { + try { + entry.getKey().get(); + } catch(ExecutionException e) { + t = e.getCause(); + } + cleanup.add(entry.getValue().completed); + } + for(var c : cleanup) + job.items.removeAll(c); + if(nofile.size() > 0 && configuration.capabilities.contains(Capabilities.VERBOSE_XML_ERROR_LOG)) { + String errorFile = "xml-error-" + job.uuid + "-" + job.library + ".log"; + Logger.warn(nofile.size() + " XML file(s) have failed to be updated due to them not being present on the file system."); + Logger.warn("This is not an issue as they're not important for Plex as it reads the ratings from the database."); + Logger.warn("The files have been dumped as " + errorFile + " in the PWD."); + ErrorReports.fileReport(nofile, errorFile); + } + if(t != null) + throw Utility.rethrow(t); + Logger.info("Completed updating of XML fallback files."); + job.stage = PipelineStage.COMPLETED; + } + } diff --git a/src/main/java/updatetool/imdb/ImdbXmlWorker.java b/src/main/java/updatetool/imdb/ImdbXmlWorker.java new file mode 100644 index 0000000..fcf7aba --- /dev/null +++ b/src/main/java/updatetool/imdb/ImdbXmlWorker.java @@ -0,0 +1,79 @@ +package updatetool.imdb; + +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import org.tinylog.Logger; +import org.w3c.dom.Document; +import updatetool.common.Utility; +import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; + +class ImdbXmlWorker implements Callable { + private final List sub; + private final Collection nofile; + private final DocumentBuilder builder; + private final AtomicInteger counter; + private final Path metadataRoot; + private final int n; + final List completed = new ArrayList<>(); + + ImdbXmlWorker(List sub, DocumentBuilder builder, AtomicInteger counter, int n, Collection nofile, Path metadataRoot) { + this.sub = sub; + this.builder = builder; + this.counter = counter; + this.n = n; + this.nofile = nofile; + this.metadataRoot = metadataRoot; + } + + @Override + public Void call() throws Exception { + for(var item : sub) { + Path contents = metadataRoot.resolve(item.hash.charAt(0)+"/"+item.hash.substring(1)+".bundle/Contents"); + Path imdb = contents.resolve("com.plexapp.agents.imdb/Info.xml"); + Path combined = contents.resolve("_combined/Info.xml"); + try { + transformXML(item, imdb, builder); + transformXML(item, combined, builder); + } catch(Exception e) { + Logger.info("Uncaught exception @ XML Worker: Continuing... ({})", e.getClass().getSimpleName()); + } + int c = counter.incrementAndGet(); + if(c % 100 == 0) + Logger.info("Transforming [{}/{}]...", c, n); + completed.add(item); + } + return null; + } + + private void transformXML(ImdbMetadataResult item, Path p, DocumentBuilder builder) throws Exception { + Document document; + try(var stream = Files.newInputStream(p)) { + document = builder.parse(stream); + } catch (NoSuchFileException e) { + nofile.add(p.toAbsolutePath().toString()); + return; + } + var children = document.getDocumentElement().getChildNodes(); + for(int i = 0; i < children.getLength(); i++) { + if(children.item(i).getNodeName().equals("rating")) + children.item(i).setTextContent(Utility.doubleToOneDecimalString(item.rating)); + if(children.item(i).getNodeName().equals("rating_image")) + children.item(i).setTextContent("imdb://image.rating"); + } + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + try(var stream = Files.newOutputStream(p)) { + transformer.transform(new DOMSource(document), new StreamResult(Files.newOutputStream(p))); + } + } +} diff --git a/src/main/java/updatetool/imdb/resolvement/NewPlexAgentToImdbResolvement.java b/src/main/java/updatetool/imdb/resolvement/NewPlexAgentToImdbResolvement.java index 06aaa77..435dca6 100644 --- a/src/main/java/updatetool/imdb/resolvement/NewPlexAgentToImdbResolvement.java +++ b/src/main/java/updatetool/imdb/resolvement/NewPlexAgentToImdbResolvement.java @@ -102,6 +102,10 @@ private boolean fallbackTvdb(ImdbMetadataResult toResolve, String candidate) { return false; } + //TODO: TVDB v3 API is incapable of resolving this at the moment + if(toResolve.type == LibraryType.SERIES && fallbackTvdb.getVersion() == ApiVersion.TVDB_V3) + return false; + String oldGuid = toResolve.guid; toResolve.guid = candidate; boolean success = fallbackTvdb.resolve(toResolve); diff --git a/src/main/resources/desc/imdb-docker.ez b/src/main/resources/desc/imdb-docker.ez index 180c016..5e1206d 100644 --- a/src/main/resources/desc/imdb-docker.ez +++ b/src/main/resources/desc/imdb-docker.ez @@ -18,6 +18,7 @@ meta { | Currently available: | - NO_TV => Ignore all TV Show libraries | - NO_MOVIE => Ignore all Movie libraries + | - VERBOSE_XML_ERROR_LOG => Enable verbose XML error output logging | - DONT_THROW_ON_ENCODING_ERROR => Supress forced quits if decoding errors of extra data are encountered due to corrupt items in the library | - IGNORE_NO_MATCHING_RESOLVER_LOG => Supresses printing items that have no matching resolver to the log | - IGNORE_SCRAPER_NO_RESULT_LOG => Supresses printing web scraper no-match results that either have no rating on the IMDB website or are not allowed to be rated by anyone on the IMDB website and thus will never have ratings From 5280e3c85e75a7d3228cfc9fa976223fd5fa3c73 Mon Sep 17 00:00:00 2001 From: mynttt Date: Thu, 16 Feb 2023 07:23:59 +0100 Subject: [PATCH 2/2] 1.7.1 - reinistated TVDB v3 API --- README.md | 1 - VERSION | 2 +- build.gradle | 2 +- src/main/java/updatetool/api/Pipeline.java | 6 +- .../java/updatetool/common/Capabilities.java | 3 +- .../java/updatetool/common/ErrorReports.java | 15 ---- .../java/updatetool/imdb/ImdbPipeline.java | 49 +----------- .../java/updatetool/imdb/ImdbXmlWorker.java | 79 ------------------- src/main/resources/VERSION | 2 +- src/main/resources/desc/imdb-docker.ez | 1 - 10 files changed, 7 insertions(+), 153 deletions(-) delete mode 100644 src/main/java/updatetool/common/ErrorReports.java delete mode 100644 src/main/java/updatetool/imdb/ImdbXmlWorker.java diff --git a/README.md b/README.md index ab69f75..6358015 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,6 @@ Flag | Description :-------------------------:|:-------------------------:| `NO_TV` |Ignore all TV Show libraries `NO_MOVIE` | Ignore all Movie libraries -`VERBOSE_XML_ERROR_LOG` | Enable verbose XML error output logging `DONT_THROW_ON_ENCODING_ERROR` | Supress forced quits if decoding errors of extra data are encountered due to corrupt items in the library `IGNORE_NO_MATCHING_RESOLVER_LOG`|Supresses printing items that have no matching resolver to the log `IGNORE_SCRAPER_NO_RESULT_LOG`|Supresses printing web scraper no-match results that either have no rating on the IMDB website or are not allowed to be rated by anyone on the IMDB website and thus will never have ratings diff --git a/VERSION b/VERSION index 9dbb0c0..081af9a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.0 \ No newline at end of file +1.7.1 \ No newline at end of file diff --git a/build.gradle b/build.gradle index fca1643..40fd666 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { id 'com.github.spotbugs' version '2.0.1' } -version = '1.7.0' +version = '1.7.1' sourceCompatibility = '11' new File(projectDir, "VERSION").text = version; diff --git a/src/main/java/updatetool/api/Pipeline.java b/src/main/java/updatetool/api/Pipeline.java index 01c8dbb..54f26b0 100644 --- a/src/main/java/updatetool/api/Pipeline.java +++ b/src/main/java/updatetool/api/Pipeline.java @@ -3,7 +3,7 @@ public abstract class Pipeline { public enum PipelineStage { - CREATED, ANALYSED_DB, ACCUMULATED_META, TRANSFORMED_META, DB_UPDATED, COMPLETED + CREATED, ANALYSED_DB, ACCUMULATED_META, TRANSFORMED_META, COMPLETED } public final void invoke(T job) throws Exception { @@ -19,9 +19,6 @@ public final void invoke(T job) throws Exception { case CREATED: analyseDatabase(job); break; - case DB_UPDATED: - updateXML(job); - break; case TRANSFORMED_META: updateDatabase(job); break; @@ -34,5 +31,4 @@ public final void invoke(T job) throws Exception { public abstract void accumulateMetadata(T job) throws Exception; public abstract void transformMetadata(T job) throws Exception; public abstract void updateDatabase(T job) throws Exception; - public abstract void updateXML(T job) throws Exception; } \ No newline at end of file diff --git a/src/main/java/updatetool/common/Capabilities.java b/src/main/java/updatetool/common/Capabilities.java index 63356d7..a6dd626 100644 --- a/src/main/java/updatetool/common/Capabilities.java +++ b/src/main/java/updatetool/common/Capabilities.java @@ -9,14 +9,13 @@ public enum Capabilities { TVDB, NO_TV, NO_MOVIE, - VERBOSE_XML_ERROR_LOG, DONT_THROW_ON_ENCODING_ERROR, IGNORE_SCRAPER_NO_RESULT_LOG, IGNORE_NO_MATCHING_RESOLVER_LOG, DISABLE_SCREEN_SCRAPE, PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS; - private static final List USER_FLAGS = List.of(NO_MOVIE, NO_TV, DONT_THROW_ON_ENCODING_ERROR, VERBOSE_XML_ERROR_LOG, IGNORE_NO_MATCHING_RESOLVER_LOG, IGNORE_SCRAPER_NO_RESULT_LOG, DISABLE_SCREEN_SCRAPE, PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS); + private static final List USER_FLAGS = List.of(NO_MOVIE, NO_TV, DONT_THROW_ON_ENCODING_ERROR, IGNORE_NO_MATCHING_RESOLVER_LOG, IGNORE_SCRAPER_NO_RESULT_LOG, DISABLE_SCREEN_SCRAPE, PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS); public static List getUserFlags() { return USER_FLAGS; diff --git a/src/main/java/updatetool/common/ErrorReports.java b/src/main/java/updatetool/common/ErrorReports.java deleted file mode 100644 index 18624e4..0000000 --- a/src/main/java/updatetool/common/ErrorReports.java +++ /dev/null @@ -1,15 +0,0 @@ -package updatetool.common; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.Collection; -import updatetool.Main; - -public class ErrorReports { - - public static void fileReport(Collection nofile, String errorFile) throws IOException { - Files.write(Main.PWD.resolve(errorFile), nofile, StandardCharsets.UTF_8); - } - -} diff --git a/src/main/java/updatetool/imdb/ImdbPipeline.java b/src/main/java/updatetool/imdb/ImdbPipeline.java index 3bbee37..9a9ec30 100644 --- a/src/main/java/updatetool/imdb/ImdbPipeline.java +++ b/src/main/java/updatetool/imdb/ImdbPipeline.java @@ -3,7 +3,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; @@ -12,26 +11,21 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; -import javax.xml.parsers.DocumentBuilderFactory; import org.sqlite.SQLiteException; import org.tinylog.Logger; -import com.google.common.collect.Lists; import updatetool.api.AgentResolvementStrategy; import updatetool.api.ExportedRating; import updatetool.api.Pipeline; import updatetool.common.Capabilities; import updatetool.common.DatabaseSupport.LibraryType; -import updatetool.common.ErrorReports; import updatetool.common.KeyValueStore; import updatetool.common.SqliteDatabaseProvider; import updatetool.common.Utility; @@ -57,7 +51,6 @@ public class ImdbPipeline extends Pipeline { + "|(?agents.thetvdb:\\/\\/)" ); - private static final int LIST_PARTITIONS = 16; private static final int RETRY_N_SECONDS_IF_DB_LOCKED = 20; private static final int ABORT_DB_LOCK_WAITING_AFTER_N_RETRIES = 500; @@ -217,7 +210,7 @@ public void transformMetadata(ImdbJob job) throws Exception { public void updateDatabase(ImdbJob job) throws Exception { if(job.items.isEmpty()) { Logger.info("Nothing to update. Skipping..."); - job.stage = PipelineStage.DB_UPDATED; + job.stage = PipelineStage.COMPLETED; return; } Logger.info("Updating " + job.items.size() + " via batch request..."); @@ -238,48 +231,10 @@ public void updateDatabase(ImdbJob job) throws Exception { } } Logger.info("Batch request finished successfully. Database is now up to date!"); - job.stage = PipelineStage.DB_UPDATED; + job.stage = PipelineStage.COMPLETED; } catch(Exception e) { throw Utility.rethrow(e); } } - @Override - public void updateXML(ImdbJob job) throws Exception { - Logger.info("Updating XML fallback files for " + job.items.size() + " item(s)."); - int n = job.items.size()/LIST_PARTITIONS; - var sublists = Lists.partition(job.items, n == 0 ? 1 : n); - var factory = DocumentBuilderFactory.newInstance(); - AtomicInteger counter = new AtomicInteger(); - HashMap, ImdbXmlWorker> map = new HashMap<>(); - var nofile = Collections.synchronizedCollection(new ArrayList()); - for(var sub : sublists) { - var worker = new ImdbXmlWorker(sub, factory.newDocumentBuilder(), counter, job.items.size(), nofile, configuration.metadataRoot); - map.put(service.submit(worker), worker); - } - Throwable t = null; - List> cleanup = new ArrayList<>(); - for(var entry : map.entrySet()) { - try { - entry.getKey().get(); - } catch(ExecutionException e) { - t = e.getCause(); - } - cleanup.add(entry.getValue().completed); - } - for(var c : cleanup) - job.items.removeAll(c); - if(nofile.size() > 0 && configuration.capabilities.contains(Capabilities.VERBOSE_XML_ERROR_LOG)) { - String errorFile = "xml-error-" + job.uuid + "-" + job.library + ".log"; - Logger.warn(nofile.size() + " XML file(s) have failed to be updated due to them not being present on the file system."); - Logger.warn("This is not an issue as they're not important for Plex as it reads the ratings from the database."); - Logger.warn("The files have been dumped as " + errorFile + " in the PWD."); - ErrorReports.fileReport(nofile, errorFile); - } - if(t != null) - throw Utility.rethrow(t); - Logger.info("Completed updating of XML fallback files."); - job.stage = PipelineStage.COMPLETED; - } - } diff --git a/src/main/java/updatetool/imdb/ImdbXmlWorker.java b/src/main/java/updatetool/imdb/ImdbXmlWorker.java deleted file mode 100644 index fcf7aba..0000000 --- a/src/main/java/updatetool/imdb/ImdbXmlWorker.java +++ /dev/null @@ -1,79 +0,0 @@ -package updatetool.imdb; - -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import org.tinylog.Logger; -import org.w3c.dom.Document; -import updatetool.common.Utility; -import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult; - -class ImdbXmlWorker implements Callable { - private final List sub; - private final Collection nofile; - private final DocumentBuilder builder; - private final AtomicInteger counter; - private final Path metadataRoot; - private final int n; - final List completed = new ArrayList<>(); - - ImdbXmlWorker(List sub, DocumentBuilder builder, AtomicInteger counter, int n, Collection nofile, Path metadataRoot) { - this.sub = sub; - this.builder = builder; - this.counter = counter; - this.n = n; - this.nofile = nofile; - this.metadataRoot = metadataRoot; - } - - @Override - public Void call() throws Exception { - for(var item : sub) { - Path contents = metadataRoot.resolve(item.hash.charAt(0)+"/"+item.hash.substring(1)+".bundle/Contents"); - Path imdb = contents.resolve("com.plexapp.agents.imdb/Info.xml"); - Path combined = contents.resolve("_combined/Info.xml"); - try { - transformXML(item, imdb, builder); - transformXML(item, combined, builder); - } catch(Exception e) { - Logger.info("Uncaught exception @ XML Worker: Continuing... ({})", e.getClass().getSimpleName()); - } - int c = counter.incrementAndGet(); - if(c % 100 == 0) - Logger.info("Transforming [{}/{}]...", c, n); - completed.add(item); - } - return null; - } - - private void transformXML(ImdbMetadataResult item, Path p, DocumentBuilder builder) throws Exception { - Document document; - try(var stream = Files.newInputStream(p)) { - document = builder.parse(stream); - } catch (NoSuchFileException e) { - nofile.add(p.toAbsolutePath().toString()); - return; - } - var children = document.getDocumentElement().getChildNodes(); - for(int i = 0; i < children.getLength(); i++) { - if(children.item(i).getNodeName().equals("rating")) - children.item(i).setTextContent(Utility.doubleToOneDecimalString(item.rating)); - if(children.item(i).getNodeName().equals("rating_image")) - children.item(i).setTextContent("imdb://image.rating"); - } - Transformer transformer = TransformerFactory.newInstance().newTransformer(); - try(var stream = Files.newOutputStream(p)) { - transformer.transform(new DOMSource(document), new StreamResult(Files.newOutputStream(p))); - } - } -} diff --git a/src/main/resources/VERSION b/src/main/resources/VERSION index 9dbb0c0..081af9a 100644 --- a/src/main/resources/VERSION +++ b/src/main/resources/VERSION @@ -1 +1 @@ -1.7.0 \ No newline at end of file +1.7.1 \ No newline at end of file diff --git a/src/main/resources/desc/imdb-docker.ez b/src/main/resources/desc/imdb-docker.ez index 5e1206d..180c016 100644 --- a/src/main/resources/desc/imdb-docker.ez +++ b/src/main/resources/desc/imdb-docker.ez @@ -18,7 +18,6 @@ meta { | Currently available: | - NO_TV => Ignore all TV Show libraries | - NO_MOVIE => Ignore all Movie libraries - | - VERBOSE_XML_ERROR_LOG => Enable verbose XML error output logging | - DONT_THROW_ON_ENCODING_ERROR => Supress forced quits if decoding errors of extra data are encountered due to corrupt items in the library | - IGNORE_NO_MATCHING_RESOLVER_LOG => Supresses printing items that have no matching resolver to the log | - IGNORE_SCRAPER_NO_RESULT_LOG => Supresses printing web scraper no-match results that either have no rating on the IMDB website or are not allowed to be rated by anyone on the IMDB website and thus will never have ratings