diff --git a/.github/workflows/collect-instance-uptime.yml b/.github/workflows/collect-instance-uptime.yml new file mode 100644 index 00000000..693f08bb --- /dev/null +++ b/.github/workflows/collect-instance-uptime.yml @@ -0,0 +1,191 @@ +name: Collect Instance Uptime +on: + schedule: + # Run every 5 mins + - cron: "* * * * *" + workflow_dispatch: + +jobs: + collect-instance-uptime: + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + + - name: Define metadata + id: define-metadata + run: | + echo "branch=instance-uptime-data" >> $GITHUB_OUTPUT + echo "directory=data" >> $GITHUB_OUTPUT + + - name: Create & push branch + run: | + exist_in_remote=$(git ls-remote --heads origin $BRANCH) + if [[ -z ${exist_in_remote} ]]; then + echo "Creating $BRANCH branch" + mkdir $DIRECTORY + cp -r .git $DIRECTORY/.git + cd $DIRECTORY + user_name='github-actions[bot]' + git config user.name "$user_name" && echo "user.name: ${user_name}" + user_email='41898282+github-actions[bot]@users.noreply.github.com' + git config user.email "$user_email" && echo "user.email: ${user_email}" + git checkout --orphan $BRANCH + git reset --hard + git commit --allow-empty -m "Initializing $BRANCH branch" + git push origin $BRANCH + fi + env: + BRANCH: ${{ steps.define-metadata.outputs.branch }} + DIRECTORY: ${{ steps.define-metadata.outputs.directory }} + + - uses: actions/checkout@v4 + with: + ref: instance-uptime-data + path: ${{ steps.define-metadata.outputs.directory }} + + - name: Collect Instance Uptime + id: collect-instance-uptime + run: | + export OS_CLOUD=BIO180006_IU # Select openstack auth settings defined in ".config/openstack/clouds.yaml" + + source ~/venv/bin/activate + + instance_prefix=${PREFIX:+${PREFIX}_} + instance_basename="${instance_prefix}instance" + + # Define the JSON file paths for storing/updating uptimes + JSON_FILE=$JSON_DIR/uptime.json + JSON_FILE_TMP=$(mktemp $RUNNER_TEMP/uptime.XXXXXX.json) + + # Initialize the JSON file if it doesn't exist, with a basic structure + if [[ ! -f $JSON_FILE ]]; then + jq --null-input \ + --arg allocation_id "$OS_CLOUD" \ + '{"allocations": [{"id": $allocation_id, "instances": []}]}' > $JSON_FILE + fi + + # List all instances matching the naming pattern and process each one + openstack server list --name "^${instance_basename}-\d+" -f json | \ + jq -r '.[] | [.Name, .Status, ."OS-EXT-STS:task_state"] | @tsv' | \ + while IFS=$'\t' read -r instance_name status task_state; do + echo "instance_name [$instance_name] status [$status] task_state [$task_state]" + + # Skip the instance if it is not active + if [[ "$status" != "ACTIVE" ]]; then + continue + fi + + if [[ "$task_state" != "" ]]; then + continue + fi + + # Extract issue number + issue_number=${instance_name##*-} + echo "issue_number [$issue_number]" + + # Retrieve the IP address of the instance + instance_ip=$( + openstack server show $instance_name -c addresses -f json | \ + jq -r '.addresses.auto_allocated_network[1]' + ) + echo "instance_ip [$instance_ip]" + + # Skip the instance if the IP address could not be retrieved + if [[ "$instance_ip" == "null" ]]; then + echo "::warning ::Failed to retrieve $instance_name IP" + continue + fi + + # Notes on SSH usage: + # * Redirecting SSH standard input to /dev/null ('< /dev/null') is required to work around + # an issue where SSH breaks out of the while loop in Bash. + # Reference: https://stackoverflow.com/questions/9393038/ssh-breaks-out-of-while-loop-in-bash + + # Retrieve the instance uptime using SSH + uptime=$(ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + exouser@$instance_ip \ + 'cat /proc/uptime | awk "{print \$1}"' < /dev/null) + + if [[ $? -ne 0 ]]; then + echo "::warning ::Failed to retrieve uptime for $instance_name using IP $instance_ip" + continue + fi + + # Retrieve the startup time of the instance in yyyy-mm-dd HH:MM:SS format + startup_time=$(ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + exouser@$instance_ip \ + 'uptime -s' < /dev/null) + if [[ $? -ne 0 ]]; then + echo "::warning ::Failed to retrieve startup time for $instance_name using IP $instance_ip" + continue + fi + + # Check if the instance is already recorded in the JSON file + has_instance=$(cat $JSON_FILE | jq \ + --arg allocation_id "$OS_CLOUD" \ + --arg instance_name "$instance_name" \ + --arg startup_time "$startup_time" \ + 'any(.allocations[] | select(.id == $allocation_id) | .instances[]; .name == $instance_name)') + + # If the instance is not found, add it to the JSON file + if [[ "$has_instance" == "false" ]]; then + cat $JSON_FILE | jq \ + --arg allocation_id "$OS_CLOUD" \ + --arg instance_name "$instance_name" \ + '(.allocations[] | select(.id == $allocation_id) | .instances) + += [{"name": $instance_name, "uptimes": {}}]' > $JSON_FILE_TMP \ + && mv $JSON_FILE_TMP $JSON_FILE + fi + + # Check if there is an uptime entry for the startup time + has_startup_time=$(cat $JSON_FILE | jq \ + --arg allocation_id "$OS_CLOUD" \ + --arg instance_name "$instance_name" \ + --arg startup_time "$startup_time" \ + '(.allocations[] | select(.id == $allocation_id) | .instances[] | select(.name == $instance_name) | .uptimes | has($startup_time))') + + if [[ "$has_startup_time" == "false" ]]; then + # If the startup time is not recorded, add the uptime entry + cat $JSON_FILE | jq \ + --arg allocation_id "$OS_CLOUD" \ + --arg instance_name "$instance_name" \ + --arg startup_time "$startup_time" \ + --arg uptime "$uptime" \ + '(.allocations[] | select(.id == $allocation_id) | .instances[] | select(.name == $instance_name) | .uptimes) + += {($startup_time): $uptime|tonumber}' > $JSON_FILE_TMP \ + && mv $JSON_FILE_TMP $JSON_FILE + else + # If the startup time is already recorded, update the uptime entry + cat $JSON_FILE | jq \ + --arg allocation_id "$OS_CLOUD" \ + --arg instance_name "$instance_name" \ + --arg startup_time "$startup_time" \ + --arg uptime "$uptime" \ + '(.allocations[] | select(.id == $allocation_id) | .instances[] | select(.name == $instance_name) | .uptimes[$startup_time]) + = ($uptime|tonumber)' > $JSON_FILE_TMP \ + && mv $JSON_FILE_TMP $JSON_FILE + fi + done + + # Display the final JSON content for verification + echo "--------" + cat $JSON_FILE + echo "--------" + env: + PREFIX: ${{ vars.INSTANCE_NAME_PREFIX }} + JSON_DIR: ${{ steps.define-metadata.outputs.directory }} + + - name: Publish + uses: s0/git-publish-subdir-action@5bc6742efb946f4cba68c7a9067a31ea5631071d # develop + env: + REPO: self + BRANCH: ${{ steps.define-metadata.outputs.branch }} + FOLDER: ${{ steps.define-metadata.outputs.directory }} + SQUASH_HISTORY: true + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}