Daily Download Scraper #74
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily Download Scraper | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| scrape-github-api: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: automation | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.x' | |
| - name: Install dependencies | |
| run: pip install requests | |
| - name: Scrape data | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 30 | |
| max_attempts: 4 | |
| command: python scripts/scraper.py | |
| # - name: Change README | |
| # run: | | |
| # python scripts/total_counter.py > output_temp.txt | |
| # number=$(cat output_temp.txt) | |
| # sed -i "0,/### Total downloads:/s/### Total downloads: [0-9]\+/### Total downloads: $number/" README.md | |
| # rm output_temp.txt | |
| - name: Commit change in download_data.json | |
| run: | | |
| git config --global user.name 'github-actions' | |
| git config --global user.email 'github-actions@github.com' | |
| git add download_data.json # README.md | |
| git commit -m "daily data update" | |
| git push origin automation | |
| - name: Sync specific files to main | |
| run: | | |
| git fetch origin | |
| git checkout main | |
| git checkout automation -- download_data.json # README.md | |
| git add download_data.json # README.md | |
| - name: Commit or amend | |
| run: | | |
| log_mes=$(git log --oneline --pretty="%an %s" | head -n 1) | |
| committer=$(echo "$log_mes" | awk '{print $1}') | |
| commit_mes=$(echo "$log_mes" | awk '{$1=""; print $0}' | sed 's/^ *//g') | |
| if [[ $committer == 'github-actions' && $commit_mes == 'daily sync download data from automation branch' ]]; then | |
| git commit --amend -m "daily sync download data from automation branch" | |
| git push --force | |
| else | |
| git commit -m "daily sync download data from automation branch" | |
| git push | |
| fi |