Merge pull request #106 from mynttt/revert-api-deprecation

Revert api deprecation

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.7.1
+- reinistated TVDB v3 API as it is still functional
+
 ## 1.7.0
 - Removed transactions from native SQLite binary interaction + using batch mode on binary to hopefully mitigate further SQLite corruption issues
 - Added `PRINT_SQLITE_BINARY_EXECUTE_STATEMENTS` capability to further diagnose future potential SQLite statement/corruption issues

README.md

@@ -71,12 +71,19 @@ Name | Description
 `USE_PLEX_SQLITE_BINARY_FOR_WRITE_ACCESS`|Allows to use the non-standard Plex SQLite3 version that diverged so strongly from the vanilla flavour that write operations with vanilla SQLite3 can cause database corruptions! Set this to `true` in the docker when using a version >= 1.6.0! This is the only way to be safe from corruptions as Plex continues to diverge from compatibility with vanilla SQLite3! If you're not using a docker version make sure that this points to the `Plex Media Server/Plex SQLite` binary that is located in the main Plex folder next to the `Plex Media Server` executable.
 `OVERRIDE_DATABASE_LOCATION`|Overrides the path where UpdateTool looks for the Plex database. The database needs to be contained in this folder. Useful if a docker container uses a volume and a different path structure. ([more here](#override-the-database-location))
 `TMDB_API_KEY`|Enables TMDB Movie/Series library processing
-`TVDB_API_KEY`|Enables TVDB Series library processing using the v4 pin
+`TVDB_API_KEY`|Enables TVDB Series library processing using either the v3 legacy key or the v4 pin
 `UNLOCK_FOR_NEW_TV_AGENT`|Opt-in for libraries using the new TV Show agent. All libraries that are opted-in this way will have their ratings changed to IMDB ratings by this tool ([more here](#opt-in-for-libraries-using-the-new-tv-show-agent))
 `IGNORE_LIBS`|Ignore libraries with certain IDs ([more here](#Ignore-libraries-from-being-updated))
 `CAPABILITIES`|Custom flags for the tool ([more here](#supply-custom-capability-flags))
 `JVM_MAX_HEAP`|Only relevant for the docker. Specify max. heap allocatable by the JVM (default 256m). Can be useful if you have a really large library (40000+ items) and you run in memory related crashes. Must be specified in bytes (i.e. 256m, 1g, 2g, 512m)
 
+Deprecated variables can still be used although their usage is discouraged.
+
+### Deprecated Environment Variables
+Name | Description | Deprecation
+:-------------------------:|:-------------------------:|:-------------------------:|
+`TVDB_AUTH_STRING`|Enables TVDB Series library processing|API Key is enough for this tool to work
+
 ## Docker on UnRaid
 
 There is a template repository available now: https://github.com/mynttt/unraid-templates

VERSION

@@ -1 +1 @@
-1.7.0
+1.7.1

build.gradle

@@ -5,7 +5,7 @@ plugins {
     id 'com.github.spotbugs' version '2.0.1'
 }
 
-version = '1.7.0'
+version = '1.7.1'
 sourceCompatibility = '11'
 
 new File(projectDir, "VERSION").text = version;

src/main/java/updatetool/common/Capabilities.java

@@ -8,7 +8,7 @@ public enum Capabilities {
     TMDB, 
     TVDB, 
     NO_TV, 
-    NO_MOVIE,
+    NO_MOVIE, 
     DONT_THROW_ON_ENCODING_ERROR,
     IGNORE_SCRAPER_NO_RESULT_LOG,
     IGNORE_NO_MATCHING_RESOLVER_LOG,

src/main/java/updatetool/common/externalapis/AbstractApi.java

@@ -16,7 +16,7 @@
 public abstract class AbstractApi {
 
     public enum ApiVersion {
-        TMDB_V3, TMDB_V4, TVDB_V4;
+        TMDB_V3, TMDB_V4, TVDB_V3, TVDB_V4;
     }
 
     private final HttpClient client;

src/main/java/updatetool/common/externalapis/TmdbApiV4.java

@@ -0,0 +1,22 @@
+package updatetool.common.externalapis;
+
+import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult;
+
+//TODO: new v4 support and v3 legacy lookup
+// TMDB API v4 appears to only be used to manage the user account and user account items like personal watchlists (07.02.2022)
+
+public class TmdbApiV4 extends AbstractApi implements TmdbApi {
+
+    @Override
+    public void resolveImdbIdForItem(ImdbMetadataResult result) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public ApiVersion version() {
+        return ApiVersion.TMDB_V3;
+    }
+
+
+}

src/main/java/updatetool/common/externalapis/TvdbApiV3.java

@@ -0,0 +1,279 @@
+package updatetool.common.externalapis;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Objects;
+import org.tinylog.Logger;
+import com.google.gson.Gson;
+import com.google.gson.internal.LinkedTreeMap;
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.Option;
+import com.jayway.jsonpath.ParseContext;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import net.minidev.json.JSONArray;
+import updatetool.common.DatabaseSupport.LibraryType;
+import updatetool.common.HttpRunner;
+import updatetool.common.KeyValueStore;
+import updatetool.common.Utility;
+import updatetool.common.HttpRunner.Converter;
+import updatetool.common.HttpRunner.Handler;
+import updatetool.common.HttpRunner.HttpCodeHandler;
+import updatetool.common.HttpRunner.RunnerResult;
+import updatetool.exceptions.ApiCallFailedException;
+import updatetool.imdb.ImdbUtility;
+import updatetool.imdb.ImdbDatabaseSupport.ImdbMetadataResult;
+
+public class TvdbApiV3 extends AbstractApi implements TvdbApi {
+    private static final ParseContext CTX = JsonPath.using(Configuration.defaultConfiguration().setOptions(Option.SUPPRESS_EXCEPTIONS));
+    private static final String BASE_URL = "https://api.thetvdb.com";
+    private String authToken;
+    private final Gson gson = new Gson();
+    private final KeyValueStore cache, blacklist, cacheMovie, blacklistMovie;
+    private final HttpRunner<String, UnmarshalTvdb, ImdbMetadataResult> runner;
+    private final HttpRunner<String, String, ImdbMetadataResult> runnerMovie;
+
+    private class UnmarshalTvdb {
+        public final String Error = null;
+        private final Object data = null;
+
+        private boolean isSeries() {
+            return data instanceof LinkedTreeMap;
+        }
+
+        private boolean isEpisode() {
+            return data instanceof ArrayList;
+        }
+
+        @SuppressWarnings("rawtypes")
+        private String getImdbId() {
+            if(data == null) return null;
+            if(isSeries()) return (String) ((LinkedTreeMap) data).get("imdbId");
+            if(isEpisode()) return (String) ((LinkedTreeMap) ((ArrayList) data).get(0)).get("imdbId");
+            return null;
+        }
+    }
+
+    private class Token { String token; };
+
+    @SuppressFBWarnings("DM_EXIT")
+    public TvdbApiV3(String key, KeyValueStore cache, KeyValueStore blacklist, KeyValueStore cacheMovie, KeyValueStore blacklistMovie) throws ApiCallFailedException {
+        Logger.info("Testing TVDB API (v3) authorization apikey: {}", key);
+
+        try {
+            authToken = "Bearer " + auth(key);
+        } catch(ApiCallFailedException e) {
+            Logger.error("API Test failed: " + e.getMessage());
+            Logger.error("Legacy (v3) keys available under: https://thetvdb.com/");
+            System.exit(-1);
+        }
+
+        Logger.info("Test passed. API Key is valid.");
+        this.cache = cache;
+        this.blacklist = blacklist;
+        this.blacklistMovie = blacklistMovie;
+        this.cacheMovie = cacheMovie;
+
+        Converter<String, UnmarshalTvdb> converter = resp -> {
+            return Objects.requireNonNull(gson.fromJson(resp.body(), UnmarshalTvdb.class));
+        };
+
+        Handler<String, UnmarshalTvdb, ImdbMetadataResult> handler = (resp, res, payload) -> {
+            if(res.Error != null) {
+                Logger.error("TVDB item (v3) {} with id {} reported error: {}", payload.title, payload.extractedId, res.Error);
+                blacklist.cache(payload.extractedId, "");
+                return RunnerResult.ofSuccess(res);
+            }
+
+            String imdbId = res.getImdbId();
+
+            if(imdbId != null && !imdbId.isBlank()) {
+                cache.cache(payload.extractedId, imdbId);
+                payload.imdbId = imdbId;
+                payload.resolved = true;
+            } else {
+                blacklist.cache(payload.extractedId, "");
+                Logger.warn("TVDB item (v3) {} with id {} does not have an IMDB id associated.", payload.title, payload.extractedId);
+            }
+
+            return RunnerResult.ofSuccess(res);
+        };
+
+        Handler<String, UnmarshalTvdb, ImdbMetadataResult> handler404 = (resp, res, payload) -> {
+            blacklist.cache(payload.extractedId, "");
+            return RunnerResult.ofSuccess(res);
+        };
+
+        Converter<String, String> converterMovie = resp -> null;
+
+        Handler<String, String, ImdbMetadataResult> handlerMovie = (resp, res, payload) -> {
+            var doc = CTX.parse(resp.body());
+            String error = doc.read("$.Error");
+            if(error != null) {
+                Logger.error("TVDB movie item (v3) {} with id {} reported error: {}", payload.title, payload.extractedId, error);
+                blacklistMovie.cache(payload.extractedId, "");
+                return RunnerResult.ofSuccess(res);
+            }
+
+            String imdbId;
+            try {
+                imdbId = (String) ((JSONArray) doc.read("$..remoteids[?(@.source_id == 2)].id")).get(0);
+            } catch(Exception e) {
+                return RunnerResult.ofSuccess(res);
+            }
+
+            if(imdbId != null && !imdbId.isBlank()) {
+                cacheMovie.cache(payload.extractedId, imdbId);
+                payload.imdbId = imdbId;
+                payload.resolved = true;
+            } else {
+                blacklistMovie.cache(payload.extractedId, "");
+                Logger.warn("TVDB movie item (v3) {} with id {} does not have an IMDB id associated.", payload.title, payload.extractedId);
+            }
+
+            return RunnerResult.ofSuccess(res);
+        };
+
+        Handler<String, String, ImdbMetadataResult> handler404m = (resp, res, payload) -> {
+            blacklistMovie.cache(payload.extractedId, "");
+            return RunnerResult.ofSuccess(res);
+        };
+
+        this.runner = new HttpRunner<>(converter, HttpCodeHandler.of(Map.of(200, handler, 404, handler404)) ,"TVDB API v3", 3);
+        this.runnerMovie = new HttpRunner<>(converterMovie, HttpCodeHandler.of(Map.of(200, handlerMovie, 404, handler404m)), "TVDB API v3 (Movie)", 3);
+    }
+
+    private String auth(String key) throws ApiCallFailedException {
+        try {
+            var response = send(
+                        postJson(BASE_URL + "/login", gson.toJson(Map.of(
+                                "apikey", key)
+                                ))
+                        );
+            if(response.statusCode() != 200) {
+                Logger.error("TVDB authorization failed with code {}", response.statusCode());
+                Logger.error("This could be due to the TVDB API having issues at the moment or your credentials being wrong.");
+                Logger.error("This is the received response:");
+                Logger.error(response.body());
+                Logger.error("===================================================");
+                throw new ApiCallFailedException("TVDB API authorization failed.");
+            }
+            return new Gson().fromJson(response.body(), Token.class).token;
+        } catch (IOException | InterruptedException e) {
+            throw Utility.rethrow(e);
+        }
+    }
+
+    private HttpResponse<String> movieImdbId(String tvdbId) {
+        try {
+            return send(HttpRequest.newBuilder(new URI(String.format("%s/movies/%s", BASE_URL, tvdbId)))
+                    .GET()
+                    .header("Authorization", authToken)
+                    .build());
+        } catch (IOException | InterruptedException | URISyntaxException e) {
+            throw Utility.rethrow(e);
+        }
+    }
+
+    private HttpResponse<String> seriesImdbId(String tvdbId) {
+        try {
+            return send(HttpRequest.newBuilder(new URI(String.format("%s/series/%s", BASE_URL, tvdbId)))
+                    .GET()
+                    .header("Authorization", authToken)
+                    .build());
+        } catch (IOException | InterruptedException | URISyntaxException e) {
+            throw Utility.rethrow(e);
+        }
+    }
+
+    private HttpResponse<String> episodeImdbId(String[] parts) {
+        try {
+            return send(HttpRequest.newBuilder(new URI(String.format("%s/series/%s/episodes/query?airedSeason=%s&airedEpisode=%s", BASE_URL, parts[0], parts[1], parts[2])))
+                    .GET()
+                    .header("Authorization", authToken)
+                    .build());
+        } catch (IOException | InterruptedException | URISyntaxException e) {
+            throw Utility.rethrow(e);
+        }
+    }
+
+    private int categorize(String tvdbId) {
+        if(ImdbUtility.TVDB_TMDB_EPISODE.matcher(tvdbId).find())
+            return 2;
+        if(ImdbUtility.TVDB_TMDB_SEASON.matcher(tvdbId).find())
+            return 1;
+        if(ImdbUtility.TVDB_TMDB_SERIES.matcher(tvdbId).find())
+            return 0;
+        throw new IllegalArgumentException("This should never happen! Input was: " + tvdbId);
+    }
+
+    @Override
+    public void resolveImdbIdForItem(ImdbMetadataResult result) {
+        if(result.type == LibraryType.MOVIE) {
+            result.extractedId = ImdbUtility.extractId(ImdbUtility.TVDB_TMDB_SERIES, result.guid);
+
+            if(result.extractedId == null) {
+                Logger.error("Item: {} is detected as TVDB Movie (v3) but has no id. (guid={})", result.title, result.guid);
+                return;
+            }
+
+            var lookup = cacheMovie.lookup(result.extractedId);
+            if(lookup != null) {
+                result.imdbId = lookup;
+                result.resolved = true;
+                return;
+            }
+
+            if(blacklistMovie.lookup(result.extractedId) != null) {
+                return;
+            }
+
+            runnerMovie.run(() -> movieImdbId(result.extractedId), result);
+        } else {
+            result.extractedId = ImdbUtility.extractId(ImdbUtility.TVDB_TMDB_SERIES_MATCHING, result.guid);
+
+            if(result.extractedId == null) {
+                Logger.error("Item: {} is detected as TVDB (v3) but has no id. (guid={})", result.title, result.guid);
+                return;
+            }
+
+            var lookup = cache.lookup(result.extractedId);
+            if(lookup != null) {
+                result.imdbId = lookup;
+                result.resolved = true;
+                return;
+            }
+
+            if(blacklist.lookup(result.extractedId) != null) {
+                return;
+            }
+
+            String[] parts = result.extractedId.split("/");
+
+            switch(categorize(result.extractedId)) {
+                case 0:
+                    runner.run(() -> seriesImdbId(parts[0]), result);
+                    break;
+                case 1:
+                 // Seasons: If ever added to IMDB could be implemented here
+                    result.resolved = false;
+                    return;
+                case 2:
+                    runner.run(() -> episodeImdbId(parts), result);
+                    break;
+                default:
+                    throw new UnsupportedOperationException();
+            }
+        }
+    }
+
+    @Override
+    public ApiVersion version() {
+        return ApiVersion.TVDB_V3;
+    }
+}