Merge pull request #38 from decentraland/fix/polygon-batch-memory

fix: polygon tracking collectionIds in memory

Author: juanmahidalgo

run.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# Load environment variables from .env file if it exists
+if [ -f .env ]; then
+  export $(grep -v '^#' .env | xargs)
+fi
+
+# Generate a unique schema name and user credentials using a timestamp
+CURRENT_TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
+NEW_SCHEMA_NAME="marketplace_squid_${CURRENT_TIMESTAMP}"
+NEW_DB_USER="marketplace_squid_user_${CURRENT_TIMESTAMP}"
+
+# Check if required environment variables are set
+if [ -z "$DB_USER" ] || [ -z "$DB_NAME" ] || [ -z "$DB_PASSWORD" ] || [ -z "$DB_HOST" ] || [ -z "$DB_PORT" ]; then
+  echo "Error: Required environment variables are not set."
+  echo "Ensure DB_USER, DB_NAME, DB_PASSWORD, DB_HOST, and DB_PORT are set."
+  exit 1
+fi
+
+# Log the generated variables
+echo "Generated schema name: $NEW_SCHEMA_NAME"
+echo "Generated user: $NEW_DB_USER"
+
+# Set PGPASSWORD to handle password prompt
+export PGPASSWORD=$DB_PASSWORD
+
+# Connect to the database and create the new schema and user
+psql -v ON_ERROR_STOP=1 --username "$DB_USER" --dbname "$DB_NAME" --host "$DB_HOST" --port "$DB_PORT" <<-EOSQL
+  CREATE SCHEMA $NEW_SCHEMA_NAME;
+  CREATE USER $NEW_DB_USER WITH PASSWORD '$DB_PASSWORD';
+  GRANT ALL PRIVILEGES ON SCHEMA $NEW_SCHEMA_NAME TO $NEW_DB_USER;
+  GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $NEW_DB_USER;
+  ALTER USER $NEW_DB_USER SET search_path TO $NEW_SCHEMA_NAME;
+EOSQL
+
+# Unset PGPASSWORD
+unset PGPASSWORD
+
+# Construct the DB_URL with the new user
+export DB_URL=postgresql://$NEW_DB_USER:$DB_PASSWORD@$DB_HOST:$DB_PORT/$DB_NAME
+export DB_SCHEMA=$NEW_SCHEMA_NAME
+
+# Log the constructed DB_URL
+echo "Exported DB_SCHEMA: $DB_SCHEMA"
+
+# Start the processor service and the GraphQL server, and write logs to a file
+LOG_FILE="sqd_run_log_${CURRENT_TIMESTAMP}.txt"
+echo "Starting squid services..."
+sqd run . > "$LOG_FILE" 2>&1
+
+echo "Logs are being written to $LOG_FILE"

src/common/modules/count.ts

@@ -12,7 +12,6 @@ export function buildCount(
 
   if (!count) {
     console.log("count not found, creating new one");
-    console.log("id: ", id);
     count = new Count({ id });
     count.orderTotal = 0;
     count.orderParcel = 0;

src/eth/handlers/bid.ts

@@ -128,7 +128,7 @@ export async function handleBidAccepted(
       counts
     );
   } else {
-    console.log("NFT not found for bid in accepted: ", id);
+    console.log("ERROR: NFT not found for bid in accepted: ", id);
   }
 }
 

src/eth/handlers/parcel.ts

@@ -36,10 +36,6 @@ export function handleUpdate(
   }
   parcel.rawData = data;
   const parcelData = buildData(id, data, DataType.PARCEL);
-  if (id === "estate-0x959e104e1a4db6317fa58f8295f586e1a978c297-10") {
-    console.log("parcelData: ", parcelData);
-    console.log("parcelId: ", parcelId);
-  }
   if (parcelData) {
     parcel.data = parcelData;
     datas.set(parcelData.id, parcelData);

src/eth/modules/analytics.ts

@@ -34,7 +34,7 @@ export function trackSale(
 ): void {
   // ignore zero price sales
   if (price === BigInt(0)) {
-    console.log("ignoring zero price sale");
+    console.log("INFO: ignoring zero price sale");
     return;
   }
 
@@ -58,7 +58,7 @@ export function trackSale(
     sale.searchContractAddress = nft.contractAddress;
     sale.searchCategory = nft.category;
   } else {
-    console.log("NFT not found for sale", nftId);
+    console.log("ERROR: NFT not found for sale", nftId);
   }
   sale.timestamp = timestamp;
   sale.txHash = txHash;

src/polygon/handlers/collection.ts

@@ -115,7 +115,7 @@ export function handleSetGlobalMinter(
 
   if (!collection) {
     console.log(
-      `Error: Collection not found in handleSetGlobalMinter: ${collectionAddress}`
+      `ERROR: Collection not found in handleSetGlobalMinter: ${collectionAddress}`
     );
     return;
   }
@@ -147,7 +147,7 @@ export function handleSetGlobalMinter(
             item.firstListedAt = BigInt(block.timestamp / 1000);
           }
         } else {
-          console.log(`Item not found: ${itemId}`);
+          console.log(`ERROR: Item not found: ${itemId}`);
         }
       }
     }
@@ -180,7 +180,7 @@ export function handleSetGlobalMinter(
           // set flag only if store is item minter, otherwise unset it
           item.searchIsStoreMinter = isStoreItemMinter;
         } else {
-          console.log(`Item not found: ${itemId}`);
+          console.log(`ERROR: Item not found: ${itemId}`);
         }
       }
     }
@@ -199,7 +199,7 @@ export function handleSetGlobalManager(
 
   if (!collection) {
     console.log(
-      `Error: Collection not found in handleSetGlobalManager: ${collectionAddress}`
+      `ERROR: Collection not found in handleSetGlobalManager: ${collectionAddress}`
     );
     return;
   }
@@ -237,7 +237,7 @@ export function handleSetItemMinter(
 
   const item = items.get(id);
   if (!item) {
-    console.log(`Error: Item not found in handleSetItemMinter: ${id}`);
+    console.log(`ERROR: Item not found in handleSetItemMinter: ${id}`);
     return;
   }
 
@@ -261,7 +261,7 @@ export function handleSetItemMinter(
     const collection = collections.get(item.collection.id);
     if (!collection) {
       console.log(
-        `Error: Collection not found in handleSetItemMinter: ${item.collection.id}`
+        `ERROR: Collection not found in handleSetItemMinter: ${item.collection.id}`
       );
     }
     if (
@@ -285,7 +285,7 @@ export function handleSetItemManager(
 
   let item = items.get(id);
   if (!item) {
-    console.log(`Error: Item not found in handleSetItemManager: ${id}`);
+    console.log(`ERROR: Item not found in handleSetItemManager: ${id}`);
     return;
   }
 
@@ -334,7 +334,7 @@ export async function handleAddItem(
     // Skip it, collection will be set up once the proxy event is created
     // The ProxyCreated event is emitted right after the collection's event
     console.log(
-      `Error: Collection not found in handleAddItem: ${collectionAddress}`
+      `ERROR: Collection not found in handleAddItem: ${collectionAddress}`
     );
     return;
   }
@@ -357,18 +357,18 @@ export async function handleAddItem(
   const rarity = rarities.get(contractItem.rarity);
   //   console.log("rarity inside Add Item: ", rarity);
   if (!rarity) {
-    console.log(`Error: Rarity not found: ${contractItem.rarity}`);
+    console.log(`ERROR: Rarity not found: ${contractItem.rarity}`);
     // return;
   }
 
   let creationFee = BigInt(0);
 
   if (!rarity) {
-    console.log("Undefined rarity {} for collection {} and item {}", [
-      contractItem.rarity,
-      collectionAddress,
-      itemId.toString(),
-    ]);
+    console.log(
+      `ERROR: Undefined rarity ${
+        contractItem.rarity
+      } for collection ${collectionAddress} and item ${itemId.toString()}`
+    );
   } else {
     creationFee = rarity.price;
 
@@ -707,7 +707,7 @@ export function handleSetApproved(
 
   if (!collection) {
     console.log(
-      `Error: Collection not found in handleSetApproved: ${collectionAddress}`
+      `ERROR: Collection not found in handleSetApproved: ${collectionAddress}`
     );
     return;
   }
@@ -805,7 +805,7 @@ export function handleSetEditable(
     collection.isEditable = event._newValue;
   } else {
     console.log(
-      `Error: Collection not found in handleSetEditable: ${collectionAddress}`
+      `ERROR: Collection not found in handleSetEditable: ${collectionAddress}`
     );
   }
 }
@@ -820,7 +820,7 @@ export function handleCompleteCollection(
     collection.isCompleted = true;
   } else {
     console.log(
-      `Error: Collection not found in handleCompleteCollection: ${collectionAddress}`
+      `ERROR: Collection not found in handleCompleteCollection: ${collectionAddress}`
     );
   }
 }
@@ -852,7 +852,7 @@ export function handleTransferCreatorship(
     }
   } else {
     console.log(
-      `Error: Collection not found in handleTransferCreatorship: ${collectionAddress}`
+      `ERROR: Collection not found in handleTransferCreatorship: ${collectionAddress}`
     );
   }
 }
@@ -868,7 +868,7 @@ export function handleTransferOwnership(
     collection.owner = event.newOwner;
   } else {
     console.log(
-      `Error: Collection not found in handleTransferOwnership: ${collectionAddress}`
+      `ERROR: Collection not found in handleTransferOwnership: ${collectionAddress}`
     );
   }
 }

src/polygon/handlers/collectionManager.ts

@@ -14,13 +14,8 @@ export async function handleRaritiesSet(
 ): Promise<void> {
   const addresses = getAddresses(Network.MATIC);
   const rarityAddress = addresses.Rarity;
-  console.log("rarityAddress: ", rarityAddress);
   const raritiesWithOracleAddress = addresses.RaritiesWithOracle;
-  console.log("raritiesWithOracleAddress: ", raritiesWithOracleAddress);
-  //   let raritiesWithOracleAddress = getRaritiesWithOracleAddress();
-  //   let rarityAddress = getRarityAddress();
   const newRaritiesAddress = event._newRarities;
-  console.log("newRaritiesAddress: ", newRaritiesAddress);
 
   if (newRaritiesAddress === raritiesWithOracleAddress) {
     let raritiesWithOracle = new RaritiesWithOracleABI.Contract(
@@ -29,7 +24,6 @@ export async function handleRaritiesSet(
       raritiesWithOracleAddress
     );
     let raritiesCount = await raritiesWithOracle.raritiesCount();
-    console.log("raritiesCount: ", raritiesCount);
 
     for (let i = 0; i < raritiesCount; i++) {
       const blockchainRarity = await raritiesWithOracle.rarities(BigInt(i));
@@ -50,9 +44,7 @@ export async function handleRaritiesSet(
     }
   } else if (newRaritiesAddress === rarityAddress) {
     const rarityContract = new RarityABI.Contract(ctx, block, rarityAddress);
-    // let rarityContract = RarityABI.bind(rarityAddress);
     const raritiesCount = await rarityContract.raritiesCount();
-    console.log("raritiesCount: ", raritiesCount);
 
     for (let i = 0; i < raritiesCount; i++) {
       const blockchainRarity = await rarityContract.rarities(BigInt(i));

src/polygon/handlers/marketplace.ts

@@ -64,9 +64,6 @@ export function handleOrderCreated(
     order.marketplaceAddress = contractAddress;
     order.status = OrderStatus.open;
     order.category = category ? (category as Category) : Category.wearable;
-    if (!order.category) {
-      console.log("order.category: ", order.category);
-    }
     order.nft = nft;
     order.network = isEthereum ? NetworkModel.ethereum : NetworkModel.polygon;
     if (!isEthereum) {

src/polygon/main.ts

@@ -1,6 +1,13 @@
+import { In, Not } from "typeorm";
 import { TypeormDatabase } from "@subsquid/typeorm-store";
 import { Network } from "@dcl/schemas";
-import { Order, Rarity, Transfer, Network as ModelNetwork } from "../model";
+import {
+  Order,
+  Rarity,
+  Transfer,
+  Network as ModelNetwork,
+  Collection,
+} from "../model";
 import * as CollectionFactoryABI from "./abi/CollectionFactory";
 import * as CollectionFactoryV3ABI from "./abi/CollectionFactoryV3";
 import * as CollectionV2ABI from "./abi/CollectionV2";
@@ -57,10 +64,12 @@ import {
 } from "./handlers/marketplace";
 import { getNFTId } from "../common/utils";
 import { handleRaritiesSet } from "./handlers/collectionManager";
+import { loadCollections } from "./utils/loaders";
 
 const schemaName = process.env.DB_SCHEMA;
 const addresses = getAddresses(Network.MATIC);
 let bytesRead = 0; // amount of bytes received
+const preloadedCollections = loadCollections().addresses;
 const collectionsCreatedByFactory = new Set<string>();
 
 processor.run(
@@ -81,6 +90,15 @@ processor.run(
       .find(Rarity)
       .then((q) => new Map(q.map((i) => [i.id, i])));
 
+    const collectionIdsNotIncludedInPreloaded = await ctx.store
+      .find(Collection, {
+        where: {
+          id: Not(In(preloadedCollections)),
+        },
+      })
+      .then((q) => new Set(q.map((c) => c.id)));
+
+    const collectionIdsCreatedInBatch = new Set<string>();
     const inMemoryData = getBatchInMemoryState();
     const {
       sales,
@@ -101,7 +119,11 @@ processor.run(
       committeeEvents,
     } = inMemoryData;
 
-    ctx.log.info(`blocks, ${ctx.blocks.length}`);
+    ctx.log.info(
+      `blocks, amount: ${ctx.blocks.length}, from: ${
+        ctx.blocks[0].header.height
+      } to: ${ctx.blocks[ctx.blocks.length - 1].header.height}`
+    );
     for (let block of ctx.blocks) {
       for (let log of block.logs) {
         const topic = log.topics[0];
@@ -124,7 +146,8 @@ processor.run(
                 ? CollectionFactoryABI.events.ProxyCreated.decode(log)
                 : CollectionFactoryV3ABI.events.ProxyCreated.decode(log);
 
-            collectionsCreatedByFactory.add(event._address.toLowerCase());
+            // collectionsCreatedByFactory.add(event._address.toLowerCase());
+            collectionIdsCreatedInBatch.add(event._address); // add the Id to the list of collections to be processed
 
             const collectionContract = new CollectionV2ABI.Contract(
               ctx,
@@ -359,7 +382,9 @@ processor.run(
                 .map((c) => c.toLowerCase())
                 .includes(log.address)
             ) {
-              console.log("Committee event found not from committee contract");
+              console.log(
+                "ERROR: Committee event found not from committee contract"
+              );
               break;
             }
             const event = CommitteeABI.events.MemberSet.decode(log);
@@ -382,7 +407,13 @@ processor.run(
           case CollectionV2ABI.events.OwnershipTransferred.topic:
           case CollectionV2ABI.events.Transfer.topic: {
             // @TODO check addresses
-            if (!collectionsCreatedByFactory.has(log.address)) {
+            if (
+              ![
+                ...preloadedCollections, // collections already pre-calculated
+                ...collectionIdsNotIncludedInPreloaded, // collections not included in the preloaded list but yes in the db (newest ones)
+                ...collectionIdsCreatedInBatch, // collections created in the current batch, will later by saved in the db
+              ].includes(log.address)
+            ) {
               // ctx.log.warn(
               //   `CollectionV2 event found not from collection contract, address: ${log.address}, topic: ${topic}, block: ${block.header.height}`
               // );
@@ -487,7 +518,7 @@ processor.run(
                 ),
               });
             } else {
-              console.log("Error: Event not decoded correctly");
+              console.log("ERROR: Event not decoded correctly");
             }
             break;
           }

src/polygon/modules/analytics.ts

@@ -75,8 +75,6 @@ export function trackSale(
     item = nft.item; // set the item coming out from the NFT
   }
   if (!item || !nft) {
-    console.log("item: ", item);
-    console.log("nft: ", nft);
     console.log(`ERROR: NFT or Item not found for sale ${nftId} ${itemId}`);
     return;
   }