Merge pull request #362 from planetarium/release/2.2.1

Release/2.2.1

common/alembic/versions/5e7148bfcf36_mileage_history_table.py

@@ -0,0 +1,36 @@
+"""Mileage history table
+
+Revision ID: 5e7148bfcf36
+Revises: b3b8a0d4eba5
+Create Date: 2024-12-05 09:43:59.814617
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '5e7148bfcf36'
+down_revision = 'b3b8a0d4eba5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('mileage_history',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('agent_addr', sa.Text(), nullable=False),
+    sa.Column('planet_id', sa.LargeBinary(length=12), nullable=True),
+    sa.Column('mileage', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('mileage_history')
+    # ### end Alembic commands ###

common/alembic/versions/bbbcfcbe1487_delete_planet_id_to_merge_mileage.py

@@ -0,0 +1,36 @@
+"""Delete planet_id to merge mileage
+
+Revision ID: bbbcfcbe1487
+Revises: 5e7148bfcf36
+Create Date: 2024-12-05 09:45:18.213511
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'bbbcfcbe1487'
+down_revision = '5e7148bfcf36'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint('unique_planet_agent', 'mileage', type_='unique')
+    op.drop_index('ix_mileage_agent_planet', table_name='mileage')
+    op.create_index('ix_mileage_agent_planet', 'mileage', ['agent_addr'], unique=False)
+    op.create_unique_constraint('unique_agent', 'mileage', ['agent_addr'])
+    op.drop_column('mileage', 'planet_id')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('mileage', sa.Column('planet_id', postgresql.BYTEA(), autoincrement=False, nullable=False))
+    op.drop_constraint('unique_agent', 'mileage', type_='unique')
+    op.drop_index('ix_mileage_agent_planet', table_name='mileage')
+    op.create_index('ix_mileage_agent_planet', 'mileage', ['planet_id', 'agent_addr'], unique=False)
+    op.create_unique_constraint('unique_planet_agent', 'mileage', ['planet_id', 'agent_addr'])
+    # ### end Alembic commands ###

common/models/mileage.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Text, Column, LargeBinary, Integer, Index, UniqueConstraint
+from sqlalchemy import Text, Column, LargeBinary, Integer, Index, UniqueConstraint, DateTime, func
 
 from common.models.base import AutoIdMixin, TimeStampMixin, Base
 from common.utils.receipt import PlanetID
@@ -7,11 +7,20 @@
 class Mileage(AutoIdMixin, TimeStampMixin, Base):
     __tablename__ = "mileage"
     agent_addr = Column(Text, nullable=False)
-    planet_id = Column(LargeBinary(length=12), nullable=False, default=PlanetID.ODIN.value,
-                       doc="An identifier of planets")
     mileage = Column(Integer, nullable=False, default=0, doc="Current mileage balance")
 
     __table_args__ = (
-        Index("ix_mileage_agent_planet", planet_id, agent_addr),
-        UniqueConstraint("planet_id", "agent_addr", name="unique_planet_agent"),
+        Index("ix_mileage_agent_planet", agent_addr),
+        UniqueConstraint("agent_addr", name="unique_agent"),
     )
+
+
+class MileageHistory(Base):
+    __tablename__ = "mileage_history"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    agent_addr = Column(Text, nullable=False)
+    planet_id = Column(LargeBinary(length=12), nullable=True, default=PlanetID.ODIN.value,
+                       doc="An identifier of planets")
+    mileage = Column(Integer, nullable=False, default=0, doc="Current mileage balance")
+    created_at = Column(DateTime(timezone=True), default=func.now())
+    updated_at = Column(DateTime(timezone=True), default=func.now(), onupdate=func.now())

common/utils/transaction.py

@@ -16,7 +16,7 @@ def create_unsigned_tx(planet_id: PlanetID, public_key: str, address: str, nonce
         # Genesis block hash
         b"g": get_genesis_block_hash(planet_id),
         # GasLimit (see also GasLimit list section below)
-        b"l": 1,
+        b"l": 4,
         # MaxGasPrice (see also Mead section for the currency spec)
         b"m": [
             {"decimalPlaces": b"\x12", "minters": None, "ticker": "Mead"},

iap/api/mileage.py

@@ -14,4 +14,6 @@
 
 @router.get("", response_model=MileageSchema)
 def get_mileage(agent_addr: str, planet_id: str, sess: Session = Depends(session)):
-    return get_mileage_fn(sess, PlanetID(bytes(planet_id, "utf-8")), agent_addr)
+    mileage = get_mileage_fn(sess, agent_addr)
+    schema = MileageSchema(**mileage.__dict__, planet_id=PlanetID(bytes(planet_id, "utf-8")))
+    return schema

iap/api/purchase.py

@@ -606,7 +606,7 @@ def mileage_product(receipt_data: FreeReceiptSchema,
         raise_error(sess, receipt, ValueError(f"Not in product opening time"))
 
     # Fetch and validate mileage
-    target_mileage = get_mileage(sess, receipt_data.planetId, receipt_data.agentAddress)
+    target_mileage = get_mileage(sess, receipt_data.agentAddress)
 
     if target_mileage.mileage < product.mileage_price:
         receipt.status = ReceiptStatus.NOT_ENOUGH_MILEAGE

iap/schemas/mileage.py

@@ -22,6 +22,7 @@ def __post_init__(self):
 
 class MileageSchema(BaseSchema):
     id: int
+    planet_id: PlanetID
     agent_addr: str
     mileage: int
 

iap/schemas/product.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from typing import Optional, List
 
 from pydantic import BaseModel as BaseSchema, model_validator
@@ -64,6 +65,8 @@ class ProductSchema(SimpleProductSchema):
     path: str
     bg_path: Optional[str] = None
     popup_path_key: Optional[str] = None
+    open_timestamp: Optional[datetime] = None
+    close_timestamp: Optional[datetime] = None
 
     fav_list: List[FungibleAssetValueSchema]
     fungible_item_list: List[FungibleItemSchema]

iap/utils.py

@@ -100,7 +100,7 @@ def create_season_pass_jwt() -> str:
     }, settings.SEASON_PASS_JWT_SECRET, algorithm="HS256")
 
 
-def get_mileage(sess, planet_id: PlanetID, agent_addr: str) -> Mileage:
+def get_mileage(sess, agent_addr: str) -> Mileage:
     """
     Read or create Mileage instance from DB.
     If no valid Mileage instance found, create new one.
@@ -111,9 +111,11 @@ def get_mileage(sess, planet_id: PlanetID, agent_addr: str) -> Mileage:
     :return: Found/created Mileage instance.
     """
     agent_addr = format_addr(agent_addr)
-    mileage = sess.scalar(select(Mileage).where(Mileage.planet_id == planet_id, Mileage.agent_addr == agent_addr))
+    # UPDATE: mileage has been merge across planets. Use one without planet_id.
+    #  Merged mileage has planet_id as None. Others are historical data.
+    mileage = sess.scalar(select(Mileage).where(Mileage.agent_addr == agent_addr))
     if not mileage:
-        mileage = Mileage(planet_id=planet_id, agent_addr=agent_addr, mileage=0)
+        mileage = Mileage(agent_addr=agent_addr, mileage=0)
         sess.add(mileage)
         sess.commit()
         sess.refresh(mileage)
@@ -133,7 +135,7 @@ def upsert_mileage(sess, product: Product, receipt: Receipt, mileage: Optional[M
     :return: Updated receipt instance.
     """
     if mileage is None:
-        mileage = get_mileage(sess, PlanetID(receipt.planet_id), receipt.agent_addr)
+        mileage = get_mileage(sess, receipt.agent_addr)
     target_mileage = product.mileage or 0
     if receipt.planet_id in (PlanetID.THOR, PlanetID.THOR_INTERNAL):
         target_mileage *= 5

script/merge_mileage.py

@@ -0,0 +1,57 @@
+import sys
+
+from sqlalchemy import create_engine, select, delete
+from sqlalchemy.orm import scoped_session, sessionmaker
+
+from common.models.mileage import Mileage, MileageHistory
+from common.utils.receipt import PlanetID
+
+
+def merge(db_uri: str, dry_run: bool = True):
+    print(f"Merge mileages for {db_uri.split('@')[-1]}")
+    if dry_run:
+        print("Dry run mode. This will not update DB.")
+    else:
+        proceed = input("You've selected update mode. This will update values into DB. Proceed? [y/N] : ")
+        print(proceed)
+        if proceed in ("y", "Y"):
+            pass
+        else:
+            print("Cancel")
+            return
+
+    engine = create_engine(db_uri)
+    sess = scoped_session(sessionmaker(bind=engine))
+
+    print("Delete old merged data")
+    sess.execute(delete(Mileage).where(Mileage.planet_id.is_(None)))
+
+    history_list = sess.scalars(select(MileageHistory)).fetchall()
+    print(f"Total {len(history_list)} histories found to merge.")
+
+    merged = {}
+    print(f"{len(history_list)} mileages to merge")
+
+    for i, m in enumerate(history_list):
+        target = merged.get(m.agent_addr)
+        if not target:
+            merged[m.agent_addr] = target = Mileage(agent_addr=m.agent_addr, planet_id=None, mileage=0)
+        prev = target.mileage
+        target.mileage += m.mileage
+        print(
+            f"[{i + 1:4d}/{len(history_list):4d}] [{PlanetID(m.planet_id).name}] "
+            f"{target.agent_addr}: {prev} + {m.mileage} = {target.mileage}"
+        )
+
+    if dry_run:
+        print("Dry run mode. Do not update to DB")
+    else:
+        print("Update to DB")
+        sess.add_all(merged.values())
+        sess.commit()
+
+
+if __name__ == "__main__":
+    merge(sys.argv[1],
+          dry_run=len(sys.argv) > 2 and sys.argv[2].lower() in ("dryrun", "dry_run", "dry-run", "dry")
+          )

script/move_mileage.py

@@ -0,0 +1,43 @@
+from sqlalchemy import create_engine, select, delete
+from sqlalchemy.orm import scoped_session, sessionmaker
+
+from common.models.mileage import Mileage, MileageHistory
+
+
+def move(db_uri: str, dry_run: bool = True):
+    print(f"Move mileage to history for {db_uri.split('@')[-1]}")
+    if dry_run:
+        print("Dry run. Do not move data.")
+    else:
+        proceed = input("You've selected update mode. This will update values into DB. Proceed? [y/N] : ")
+        print(proceed)
+        if proceed in ("y", "Y"):
+            pass
+        else:
+            print("Cancel")
+            return
+
+    engine = create_engine(db_uri)
+    sess = scoped_session(sessionmaker(bind=engine))
+    mileage_list = sess.scalars(select(Mileage)).fetchall()
+    history_list = []
+    print(f"{len(mileage_list)} mileages to save")
+    for m in mileage_list:
+        history_list.append(MileageHistory(**m.__dict__))
+
+    print(f"{len(history_list)}/{len(mileage_list)} moved to history")
+    if len(history_list) != len(mileage_list):
+        print("Mileage and history count not match. Abort.")
+        return
+
+    if dry_run:
+        print("Dry run. Do not save history and delete mileage.")
+    else:
+        sess.add_all(history_list)
+        sess.commit()
+        print("History saved. Delete mileage data")
+        sess.execute(delete(Mileage))
+        sess.commit()
+        print("Mileage deleted. Merge mileage from history.")
+
+

tests/utils/test_transaction.py

@@ -75,7 +75,7 @@ def test_create_unsigned_tx():
         datetime.datetime(2021, 10, 1, 5, 36, 33, 194530)
     )
 
-    expected_tx = "64313a616c6475373a747970655f69647531313a636c61696d5f6974656d7375363a76616c7565736475323a63646c6c32303ae27560fc2e0cd920633eddbcbf09e703a75827496c6c647531333a646563696d616c506c61636573313a0075373a6d696e746572736e75363a7469636b65727531343a4974656d5f4e545f353030303030656931656565656575323a696431363a3979aba1631d25438c8bf93eed328eff75313a6d7535303a7b22736561736f6e5f70617373223a207b226e223a205b315d2c202270223a205b5d2c202274223a2022636c61696d227d7d656565313a6733323a4582250d0da33b06779a8475d283d5dd210c683b9b999d74d03fac4f58fa6bce313a6c693165313a6d6c647531333a646563696d616c506c61636573313a1275373a6d696e746572736e75363a7469636b657275343a4d6561646569313030303030303030303030303030303030306565313a6e693065313a7033333a024007a3342b03083e7c87e80b6daa9be6f7e1caae66c368fb32ca43994394ef3e313a7332303a8ba11bef1db41f3118f7478ccfcbe7f1af4650fa313a747532373a323032312d31302d30315430353a33363a33332e3139343533305a313a756c6565"
+    expected_tx = "64313a616c6475373a747970655f69647531313a636c61696d5f6974656d7375363a76616c7565736475323a63646c6c32303ae27560fc2e0cd920633eddbcbf09e703a75827496c6c647531333a646563696d616c506c61636573313a0075373a6d696e746572736e75363a7469636b65727531343a4974656d5f4e545f353030303030656931656565656575323a696431363a3979aba1631d25438c8bf93eed328eff75313a6d7535303a7b22736561736f6e5f70617373223a207b226e223a205b315d2c202270223a205b5d2c202274223a2022636c61696d227d7d656565313a6733323a4582250d0da33b06779a8475d283d5dd210c683b9b999d74d03fac4f58fa6bce313a6c693465313a6d6c647531333a646563696d616c506c61636573313a1275373a6d696e746572736e75363a7469636b657275343a4d6561646569313030303030303030303030303030303030306565313a6e693065313a7033333a024007a3342b03083e7c87e80b6daa9be6f7e1caae66c368fb32ca43994394ef3e313a7332303a8ba11bef1db41f3118f7478ccfcbe7f1af4650fa313a747532373a323032312d31302d30315430353a33363a33332e3139343533305a313a756c6565"
     loaded_tx = bencodex.loads(unsigned_tx)
 
     assert expected_tx == unsigned_tx.hex()