diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml
index 372e903e6d85d..b4ddffdb9dbb7 100644
--- a/.github/workflows/weekly.yml
+++ b/.github/workflows/weekly.yml
@@ -12,6 +12,7 @@ env:
   CARGO_INCREMENTAL: 0
   CARGO_PROFILE_TEST_DEBUG: 0
   CARGO_PROFILE_DEV_DEBUG: 0
+  ISSUE_TITLE: Main branch fails to compile on Rust beta.
 
 # The jobs listed here are intentionally skipped when running on forks, for a number of reasons:
 #
@@ -82,6 +83,30 @@ jobs:
         # See tools/ci/src/main.rs for the commands this runs
         run: cargo run -p ci -- compile
 
+  close-any-open-issues:
+    runs-on: ubuntu-latest
+    needs: ['test', 'lint', 'check-compiles']
+    permissions:
+      issues: write
+    steps:
+      - name: Close issues
+        run: |
+          previous_issue_number=$(gh issue list \
+            --search "$ISSUE_TITLE in:title" \
+            --json number \
+            --jq '.[0].number')
+          if [[ -n $previous_issue_number ]]; then
+            gh issue close $previous_issue_number \
+              -r completed \
+              -c $COMMENT
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          COMMENT: |
+            [Last pipeline run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) successfully completed. Closing issue.
+
+
   open-issue:
     name: Warn that weekly CI fails
     runs-on: ubuntu-latest
@@ -95,7 +120,7 @@ jobs:
       - name: Create issue
         run: |
           previous_issue_number=$(gh issue list \
-            --search "$TITLE in:title" \
+            --search "$ISSUE_TITLE in:title" \
             --json number \
             --jq '.[0].number')
           if [[ -n $previous_issue_number ]]; then
@@ -103,14 +128,13 @@ jobs:
               --body "Weekly pipeline still fails: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           else
             gh issue create \
-              --title "$TITLE" \
+              --title "$ISSUE_TITLE" \
               --label "$LABELS" \
               --body "$BODY"
           fi
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GH_REPO: ${{ github.repository }}
-          TITLE: Main branch fails to compile on Rust beta.
           LABELS: C-Bug,S-Needs-Triage
           BODY: |
             ## Weekly CI run has failed.