A lot of code iterates the allprintings file. Simplify that code a bi…

…t by moving redundant components (mtgjson#647)

mtgjson5/providers/cardhoarder.py

@@ -1,16 +1,15 @@
 """
 CardHoarder 3rd party provider
 """
-import json
 import logging
 import pathlib
-from typing import Any, Dict, Iterator, List, Union
+from typing import Any, Dict, List, Union
 
 from singleton_decorator import singleton
 
 from ..classes import MtgjsonPricesObject
 from ..providers.abstract import AbstractProvider
-from ..utils import retryable_session
+from ..utils import get_all_cards_and_tokens, retryable_session
 
 LOGGER = logging.getLogger(__name__)
 
@@ -152,31 +151,14 @@ def _construct_for_cards(
                 semi_completed_data[key].sell_foil = float(value)
 
     @staticmethod
-    def iterate_cards_and_tokens(
-        all_printings_path: pathlib.Path,
-    ) -> Iterator[Dict[str, Any]]:
-        """
-        Grab every card and token object from an AllPrintings file for future iteration
-        :param all_printings_path: AllPrintings.json to refer when building
-        :return Iterator for all card and token objects
-        """
-        with all_printings_path.expanduser().open(encoding="utf-8") as f:
-            file_contents = json.load(f).get("data", {})
-
-        for value in file_contents.values():
-            for card in value.get("cards", []) + value.get("tokens", []):
-                yield card
-
-    def get_mtgo_to_mtgjson_map(
-        self, all_printings_path: pathlib.Path
-    ) -> Dict[str, str]:
+    def get_mtgo_to_mtgjson_map(all_printings_path: pathlib.Path) -> Dict[str, str]:
         """
         Construct a mapping from MTGO IDs (Regular & Foil) to MTGJSON UUIDs
         :param all_printings_path: AllPrintings to generate mapping from
         :return MTGO to MTGJSON mapping
         """
         mtgo_to_mtgjson = dict()
-        for card in self.iterate_cards_and_tokens(all_printings_path):
+        for card in get_all_cards_and_tokens(all_printings_path):
             identifiers = card["identifiers"]
             if "mtgoId" in identifiers:
                 mtgo_to_mtgjson[identifiers["mtgoId"]] = card["uuid"]

mtgjson5/providers/cardkingdom.py

@@ -1,7 +1,6 @@
 """
 Card Kingdom 3rd party provider
 """
-import json
 import logging
 import pathlib
 from typing import Any, Dict, Union
@@ -10,7 +9,7 @@
 
 from ..classes import MtgjsonPricesObject
 from ..providers.abstract import AbstractProvider
-from ..utils import retryable_session
+from ..utils import generate_card_mapping, retryable_session
 
 LOGGER = logging.getLogger(__name__)
 
@@ -59,56 +58,39 @@ def generate_today_price_dict(
         :return MTGJSON prices single day structure
         """
         request_api_response: Dict[str, Any] = self.download(self.api_url)
-        translation_table = self.generate_card_kingdom_to_mtgjson_map(
-            all_printings_path
+
+        # Start with non-foil IDs
+        card_kingdom_id_to_mtgjson = generate_card_mapping(
+            all_printings_path, ("identifiers", "cardKingdomId"), ("uuid",)
+        )
+
+        # Then add in foil IDs
+        card_kingdom_id_to_mtgjson.update(
+            generate_card_mapping(
+                all_printings_path, ("identifiers", "cardKingdomFoilId"), ("uuid",)
+            )
         )
 
         today_dict: Dict[str, MtgjsonPricesObject] = {}
 
         card_rows = request_api_response.get("data", [])
         for card in card_rows:
             card_id = str(card["id"])
-            if card_id in translation_table.keys():
-                mtgjson_uuid = translation_table[card_id]
+            if card_id not in card_kingdom_id_to_mtgjson:
+                continue
 
-                if mtgjson_uuid not in today_dict:
-                    today_dict[mtgjson_uuid] = MtgjsonPricesObject(
-                        "paper", "cardkingdom", self.today_date
-                    )
+            mtgjson_uuid = card_kingdom_id_to_mtgjson[card_id]
 
-                if card["is_foil"] == "true":
-                    today_dict[mtgjson_uuid].sell_foil = float(card["price_retail"])
-                    today_dict[mtgjson_uuid].buy_foil = float(card["price_buy"])
-                else:
-                    today_dict[mtgjson_uuid].sell_normal = float(card["price_retail"])
-                    today_dict[mtgjson_uuid].buy_normal = float(card["price_buy"])
+            if mtgjson_uuid not in today_dict:
+                today_dict[mtgjson_uuid] = MtgjsonPricesObject(
+                    "paper", "cardkingdom", self.today_date
+                )
 
-        return today_dict
+            if card["is_foil"] == "true":
+                today_dict[mtgjson_uuid].sell_foil = float(card["price_retail"])
+                today_dict[mtgjson_uuid].buy_foil = float(card["price_buy"])
+            else:
+                today_dict[mtgjson_uuid].sell_normal = float(card["price_retail"])
+                today_dict[mtgjson_uuid].buy_normal = float(card["price_buy"])
 
-    @staticmethod
-    def generate_card_kingdom_to_mtgjson_map(
-        all_printings_path: pathlib.Path,
-    ) -> Dict[str, str]:
-        """
-        Generate a TCGPlayerID -> MTGJSON UUID map that can be used
-        across the system.
-        :param all_printings_path: Path to JSON compiled version
-        :return: Map of TCGPlayerID -> MTGJSON UUID
-        """
-        with all_printings_path.expanduser().open(encoding="utf-8") as f:
-            file_contents = json.load(f).get("data", {})
-
-        dump_map: Dict[str, str] = {}
-        for value in file_contents.values():
-            for card in value.get("cards", []) + value.get("tokens", []):
-                try:
-                    dump_map[card["identifiers"]["cardKingdomId"]] = card["uuid"]
-                except KeyError:
-                    pass
-
-                try:
-                    dump_map[card["identifiers"]["cardKingdomFoilId"]] = card["uuid"]
-                except KeyError:
-                    pass
-
-        return dump_map
+        return today_dict

mtgjson5/providers/cardmarket.py

@@ -3,7 +3,6 @@
 """
 import base64
 import io
-import json
 import logging
 import math
 import os
@@ -18,6 +17,7 @@
 
 from ..classes import MtgjsonPricesObject
 from ..providers.abstract import AbstractProvider
+from ..utils import generate_card_mapping
 
 LOGGER = logging.getLogger(__name__)
 
@@ -82,7 +82,9 @@ def generate_today_price_dict(
         if not self.__keys_found:
             return {}
 
-        mtgjson_id_map = self._generate_cardmarket_to_mtgjson_map(all_printings_path)
+        mtgjson_id_map = generate_card_mapping(
+            all_printings_path, ("identifiers", "mcmId"), ("uuid",)
+        )
 
         price_data = pandas.read_csv(self._get_card_market_data())
         data_frame_columns = list(price_data.columns)
@@ -119,29 +121,6 @@ def generate_today_price_dict(
 
         return today_dict
 
-    @staticmethod
-    def _generate_cardmarket_to_mtgjson_map(
-        all_printings_path: pathlib.Path,
-    ) -> Dict[str, str]:
-        """
-        Generate a CardMarketID -> MTGJSON UUID map that can be used
-        across the system.
-        :param all_printings_path: Path to JSON compiled version
-        :return: Map of CardMarketID -> MTGJSON UUID
-        """
-        with all_printings_path.expanduser().open(encoding="utf-8") as f:
-            file_contents = json.load(f).get("data", {})
-
-        dump_map: Dict[str, str] = {}
-        for value in file_contents.values():
-            for card in value.get("cards", []) + value.get("tokens", []):
-                try:
-                    dump_map[card["identifiers"]["mcmId"]] = card["uuid"]
-                except KeyError:
-                    pass
-
-        return dump_map
-
     def __init_set_map(self) -> None:
         """
         Construct a mapping for all set components from MKM

mtgjson5/providers/tcgplayer.py

@@ -12,7 +12,7 @@
 
 from ..classes import MtgjsonPricesObject
 from ..providers.abstract import AbstractProvider
-from ..utils import parallel_call, retryable_session
+from ..utils import generate_card_mapping, parallel_call, retryable_session
 
 LOGGER = logging.getLogger(__name__)
 
@@ -180,7 +180,9 @@ def generate_today_price_dict(
             return {}
 
         ids_and_names = self.get_tcgplayer_magic_set_ids()
-        tcg_to_mtgjson_map = generate_tcgplayer_to_mtgjson_map(all_printings_path)
+        tcg_to_mtgjson_map = generate_card_mapping(
+            all_printings_path, ("identifiers", "tcgplayerProductId"), ("uuid",)
+        )
 
         LOGGER.info("Building TCGPlayer buylist data")
         buylist_dict = parallel_call(
@@ -281,29 +283,6 @@ def get_tcgplayer_sku_map(
     return tcgplayer_sku_map
 
 
-def generate_tcgplayer_to_mtgjson_map(
-    all_printings_path: pathlib.Path,
-) -> Dict[str, str]:
-    """
-    Generate a TCGPlayerID -> MTGJSON UUID map that can be used
-    across the system.
-    :param all_printings_path: Path to JSON compiled version
-    :return: Map of TCGPlayerID -> MTGJSON UUID
-    """
-    with all_printings_path.expanduser().open(encoding="utf-8") as f:
-        file_contents = json.load(f).get("data", {})
-
-    dump_map: Dict[str, str] = {}
-    for value in file_contents.values():
-        for card in value.get("cards", []) + value.get("tokens", []):
-            try:
-                dump_map[card["identifiers"]["tcgplayerProductId"]] = card["uuid"]
-            except KeyError:
-                pass
-
-    return dump_map
-
-
 def get_tcgplayer_buylist_prices_map(
     group_id_and_name: Tuple[str, str], tcg_to_mtgjson_map: Dict[str, str]
 ) -> Dict[str, MtgjsonPricesObject]:
@@ -400,26 +379,3 @@ def get_tcgplayer_prices_map(
             prices_map[key].sell_foil = card_price
 
     return prices_map
-
-
-def generate_mtgjson_to_tcgplayer_map(
-    all_printings_path: pathlib.Path,
-) -> Dict[str, str]:
-    """
-    Generate a TCGPlayerID -> MTGJSON UUID map that can be used
-    across the system.
-    :param all_printings_path: Path to JSON compiled version
-    :return: Map of TCGPlayerID -> MTGJSON UUID
-    """
-    with all_printings_path.expanduser().open(encoding="utf-8") as f:
-        file_contents = json.load(f).get("data", {})
-
-    dump_map: Dict[str, str] = {}
-    for value in file_contents.values():
-        for card in value.get("cards", []) + value.get("tokens", []):
-            try:
-                dump_map[card["uuid"]] = card["identifiers"]["tcgplayerProductId"]
-            except KeyError:
-                pass
-
-    return dump_map

mtgjson5/utils.py

@@ -5,11 +5,12 @@
 import hashlib
 import inspect
 import itertools
+import json
 import logging
 import os
 import pathlib
 import time
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
 
 import gevent.pool
 import requests
@@ -282,3 +283,56 @@ def deep_merge_dictionaries(
     )
 
     return new_dictionary
+
+
+def get_all_cards_and_tokens(
+    all_printings_path: pathlib.Path,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Grab every card and token object from an AllPrintings file for future iteration
+    :param all_printings_path: AllPrintings.json to refer when building
+    :return Iterator for all card and token objects
+    """
+    all_printings_path = all_printings_path.expanduser()
+    if not all_printings_path.exists():
+        LOGGER.error(f"File {all_printings_path} does not exist, cannot iterate")
+        return
+
+    with all_printings_path.open(encoding="utf-8") as f:
+        file_contents = json.load(f).get("data", {})
+
+    for value in file_contents.values():
+        for card in value.get("cards", []) + value.get("tokens", []):
+            yield card
+
+
+def generate_card_mapping(
+    all_printings_path: pathlib.Path,
+    left_side_components: Tuple[str, ...],
+    right_side_components: Tuple[str, ...],
+) -> Dict[str, Any]:
+    """
+    Construct a mapping from one component of the card to another.
+    The components are nested ops to get to the final value.
+    :param all_printings_path: AllPrintings file to load card data from
+    :param left_side_components: Inner left hand side components ([foo, bar] => card[foo][bar])
+    :param right_side_components: Inner right hand side components ([foo, bar] => card[foo][bar])
+    :return Dict mapping from left components => right components
+    """
+    dump_map: Dict[str, Any] = {}
+
+    for card in get_all_cards_and_tokens(all_printings_path):
+        try:
+            key = card
+            for inside_component in left_side_components:
+                key = key[inside_component]
+
+            value = card
+            for inside_component in right_side_components:
+                value = value[inside_component]
+
+            dump_map[str(key)] = value
+        except KeyError:
+            pass
+
+    return dump_map