Merge branch 'main' into issue-1328/wise-bank-account-payee-name-field

.github/workflows/codeql-analysis.yml

@@ -27,15 +27,15 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v3
       # Override language selection by uncommenting this and choosing your languages
       # with:
       #   languages: go, javascript, csharp, python, cpp, java
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -49,4 +49,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

.ignore

@@ -0,0 +1,2 @@
+images
+exports

.mypy.ini

@@ -94,3 +94,6 @@ ignore_missing_imports = True
 
 [mypy-flask_mailman.*]
 ignore_missing_imports = True
+
+[mypy-stdnum.*]
+ignore_missing_imports = True

Dockerfile.prod

@@ -8,14 +8,13 @@ FROM ghcr.io/emfcamp/website-base:latest
 COPY . /app/
 WORKDIR /app
 
-RUN poetry install --no-dev && poetry run pyppeteer-install && mkdir /var/prometheus \
-	&& rm -Rf ./static
+RUN poetry install --no-dev && poetry run pyppeteer-install && rm -Rf ./static
 
 COPY --from=static /app/static /app/static
 
 # The settings file doesn't matter for the purposes of the below command,
 # but it's needed for it to run.
 RUN SETTINGS_FILE=/app/config/test.cfg poetry run flask digest compile
-
+ENV PROMETHEUS_MULTIPROC_DIR=/var/prometheus
 ENV SHELL=/bin/bash
 ENTRYPOINT ["./docker/prod_entrypoint.sh"]

apps/admin/__init__.py

@@ -159,8 +159,7 @@ def feature_flags():
 class SiteStateForm(Form):
     site_state = SelectField(
         "Site",
-        choices=[("", "(automatic)")]
-        + list(zip(VALID_STATES["site_state"], VALID_STATES["site_state"])),
+        choices=list(zip(VALID_STATES["site_state"], VALID_STATES["site_state"])),
     )
     sales_state = SelectField(
         "Sales",

apps/admin/accounts.py

@@ -90,6 +90,7 @@ def transaction_suggest_payments(txn_id):
 
     payments = (
         BankPayment.query.filter_by(state="inprogress")
+        .filter(BankPayment.created < txn.posted)
         .order_by(BankPayment.bankref)
         .all()
     )
@@ -148,7 +149,7 @@ def transaction_reconcile(txn_id, payment_id):
 
             already_emailed = set_tickets_emailed(payment.user)
             msg.body = render_template(
-                "emails/tickets-paid-email-banktransfer.txt",
+                "emails/payment-paid.txt",
                 user=payment.user,
                 payment=payment,
                 already_emailed=already_emailed,

apps/admin/email.py

@@ -1,3 +1,4 @@
+from models import event_year
 from . import admin
 from flask import render_template, redirect, flash, url_for
 from wtforms import SubmitField, StringField, SelectField
@@ -21,7 +22,6 @@ class EmailComposeForm(Form):
     destination = SelectField(
         "Send to:",
         choices=[
-            ("all", "Registered users"),
             ("ticket", "Ticketholders"),
             ("purchasers", "Users who made payments"),
             ("cfp", "Accepted CfP"),
@@ -46,15 +46,31 @@ def get_users(dest: str) -> list[User]:
     elif dest == "purchasers":
         query = query.join(User.payments).filter(Payment.state == "paid")
     elif dest == "cfp":
-        query = query.join(User.proposals).filter(
-            Proposal.state.in_(("accepted", "finished"))
-        )
+        query = query.join(User.proposals).filter(Proposal.is_accepted)
     elif dest == "villages":
         query = query.join(User.village_membership).filter(VillageMember.admin)
+    else:
+        raise ValueError("Invalid email destination set: %s" % dest)
 
     return query.distinct().all()
 
 
+def get_email_reason(dest: str) -> str:
+    event = f"Electromagnetic Field {event_year()}"
+    if dest == "ticket":
+        return f"You're receiving this email because you have a ticket for {event}."
+    elif dest == "purchasers":
+        return f"You're receiving this email because you made a payment to {event}."
+    elif dest == "cfp":
+        return f"You're receiving this email because you have an accepted proposal in the {event} Call for Participation."
+    elif dest == "villages":
+        return f"You're receiving this email because you have registered a village for {event}."
+    elif dest == "ticket_and_cfp":
+        return f"You're receiving this email because you have a ticket or a talk/workshop accepted for {event}."
+    else:
+        raise ValueError("Invalid email destination set: %s" % dest)
+
+
 @admin.route("/email", methods=["GET", "POST"])
 def email():
     form = EmailComposeForm()
@@ -65,10 +81,15 @@ def email():
             users.update(get_users("cfp"))
         else:
             users = get_users(form.destination.data)
+
+        reason = get_email_reason(form.destination.data)
+
         if form.preview.data is True:
             return render_template(
                 "admin/email.html",
-                html=format_trusted_html_email(form.text.data, form.subject.data),
+                html=format_trusted_html_email(
+                    form.text.data, form.subject.data, reason=reason
+                ),
                 form=form,
                 count=len(users),
             )
@@ -81,13 +102,22 @@ def email():
             flash("Email preview sent to %s" % form.send_preview_address.data)
             return render_template(
                 "admin/email.html",
-                html=format_trusted_html_email(form.text.data, form.subject.data),
+                html=format_trusted_html_email(
+                    form.text.data,
+                    form.subject.data,
+                    reason=reason,
+                ),
                 form=form,
                 count=len(users),
             )
 
         if form.send.data is True:
-            enqueue_trusted_emails(users, form.subject.data, form.text.data)
+            enqueue_trusted_emails(
+                users,
+                form.subject.data,
+                form.text.data,
+                reason=reason,
+            )
             flash("Email queued for sending to %s users" % len(users))
             return redirect(url_for(".email"))
 

apps/admin/forms.py

@@ -19,8 +19,8 @@
 from models.basket import Basket
 
 from ..common import CURRENCY_SYMBOLS
-from ..common.forms import (
-    Form,
+from ..common.forms import Form
+from ..common.fields import (
     IntegerSelectField,
     HiddenIntegerField,
     JSONField,

apps/admin/payments.py

@@ -11,6 +11,7 @@
     abort,
     current_app as app,
 )
+from flask.typing import ResponseReturnValue
 from flask_login import current_user
 from flask_mailman import EmailMessage
 
@@ -30,7 +31,8 @@
 )
 from models.purchase import AdmissionTicket, Purchase
 from ..common.email import from_email
-from ..common.forms import Form, HiddenIntegerField
+from ..common.forms import Form
+from ..common.fields import HiddenIntegerField
 from ..payments.stripe import (
     StripeUpdateUnexpected,
     StripeUpdateConflict,
@@ -583,3 +585,57 @@ def change_currency(payment_id):
         form=form,
         new_currency=new_currency,
     )
+
+
+class CancelPurchaseForm(Form):
+    cancel = SubmitField("Cancel purchase")
+
+
+@admin.route(
+    "/payment/<int:payment_id>/cancel_purchase/<int:purchase_id>",
+    methods=["GET", "POST"],
+)
+def cancel_purchase(payment_id: int, purchase_id: int) -> ResponseReturnValue:
+    """Remove a purchase from a payment before it has been paid.
+
+    This is used when the purchaser changes their mind before they've sent us the money.
+    """
+    payment: Payment = Payment.query.get_or_404(payment_id)
+    purchase: Purchase = Purchase.query.get_or_404(purchase_id)
+
+    if purchase.payment != payment:
+        return abort(400)
+
+    if purchase.state != "payment-pending":
+        return abort(400)
+
+    form = CancelPurchaseForm()
+    if form.validate_on_submit():
+        if form.cancel.data:
+            app.logger.info(
+                "%s manually cancelling purchase %s", current_user.name, purchase.id
+            )
+            payment.lock()
+
+            try:
+                purchase.cancel()
+            except StateException as e:
+                msg = "Could not cancel purchase %s: %s" % (purchase_id, e)
+                app.logger.warn(msg)
+                flash(msg)
+                return redirect(url_for("admin.payment", payment_id=payment.id))
+
+            purchase.payment_id = None
+            payment.amount -= purchase.price_tier.get_price(payment.currency).value
+
+            db.session.commit()
+
+            flash("Purchase %s cancelled" % purchase.id)
+            return redirect(url_for("admin.payment", payment_id=payment.id))
+
+    return render_template(
+        "admin/payments/purchase-cancel.html",
+        payment=payment,
+        purchase=purchase,
+        form=form,
+    )

apps/admin/products.py

@@ -26,7 +26,7 @@
     random_voucher,
     Voucher,
 )
-from models.purchase import Purchase, PurchaseTransfer
+from models.purchase import Purchase
 
 from ..common.email import (
     format_trusted_html_email,
@@ -366,14 +366,6 @@ def product_group_copy(group_id):
     )
 
 
-@admin.route("/transfers")
-def purchase_transfers():
-    transfer_logs = PurchaseTransfer.query.all()
-    return render_template(
-        "admin/products/purchase-transfers.html", transfers=transfer_logs
-    )
-
-
 @admin.route("/tees")
 def tees():
     purchases = (
@@ -458,7 +450,10 @@ def product_view(view_id):
         db.session.commit()
 
     active_vouchers = Voucher.query.filter_by(view=view).filter(
-        not_(Voucher.expiry.isnot(None) & (Voucher.expiry < func.now()))
+        not_(
+            Voucher.expiry.isnot(None)
+            & (Voucher.expiry + VOUCHER_GRACE_PERIOD < func.now())
+        )
         & not_(Voucher.is_used)
     )
     stats = {
@@ -533,7 +528,7 @@ def product_view_voucher_list(view_id):
         vouchers = vouchers.filter(
             not_(
                 Voucher.expiry.isnot(None)
-                & (Voucher.expiry < func.now() - VOUCHER_GRACE_PERIOD)
+                & (Voucher.expiry + VOUCHER_GRACE_PERIOD < func.now())
             )
         )
 

apps/admin/search.py

@@ -34,10 +34,12 @@ def search():
     # how small our dataset is, but I spent ages trying to work out how to get Alembic to add
     # those indexes. So humour me.
     results = User.query.filter(
-        func.to_tsvector("simple", User.name).match(to_query(q))
+        func.to_tsvector("simple", User.name).match(
+            to_query(q), postgresql_regconfig="simple"
+        )
         | (
             func.to_tsvector("simple", func.replace(User.email, "@", " ")).match(
-                email_query
+                email_query, postgresql_regconfig="simple"
             )
         )
     )

apps/admin/tickets.py

@@ -6,7 +6,6 @@
     flash,
     url_for,
     current_app as app,
-    abort,
     send_file,
 )
 from flask_mailman import EmailMessage
@@ -168,11 +167,18 @@ def list_free_tickets():
     )
 
 
+@admin.route("/ticket/<int:ticket_id>", methods=["GET"])
+def view_ticket(ticket_id):
+    ticket = Ticket.query.get_or_404(ticket_id)
+    return render_template(
+        "admin/tickets/view_ticket.html",
+        ticket=ticket,
+    )
+
+
 @admin.route("/ticket/<int:ticket_id>/cancel-free", methods=["GET", "POST"])
 def cancel_free_ticket(ticket_id):
     ticket = Purchase.query.get_or_404(ticket_id)
-    if ticket.payment is not None:
-        abort(404)
 
     form = CancelTicketForm()
     if form.validate_on_submit():
@@ -307,7 +313,7 @@ def tickets_reserve(email):
         )
 
         msg.body = render_template(
-            "emails/tickets-reserved.txt",
+            "emails/admin-tickets-reserved.txt",
             user=user,
             code=code,
             tickets=basket.purchases,