From 1a6323c3e8238b598dfd916a71740ae9896192f7 Mon Sep 17 00:00:00 2001 From: David Lehuby Date: Thu, 18 Jul 2024 11:36:17 +1000 Subject: [PATCH] PM-1901 - Refactor mina-staking-ledgers-exporter script to make uploads optional --- .pre-commit-config.yaml | 2 +- matterbridge-bot/README.md | 1 + mina-staking-ledgers-exporter/README.md | 116 +++++++++--------- .../scripts/mina-staking-ledgers-exporter.py | 31 +++-- .../templates/cronjob.yaml | 4 + mina-staking-ledgers-exporter/values.yaml | 39 +++++- 6 files changed, 118 insertions(+), 75 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a45ca689..1ed636e7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - id: helm-docs-built args: # Comma separated list, no space - - --chart-to-generate=gpt-survey-summarizer,submission-report,mina-transactions-generator,mina-payouts-data-provider,mina-archive,redisinsight,uptime-service-backend,liminal-manual,delegation-program-leaderboard,matterbridge-bot + - --chart-to-generate=gpt-survey-summarizer,submission-report,mina-transactions-generator,mina-payouts-data-provider,mina-archive,redisinsight,uptime-service-backend,liminal-manual,delegation-program-leaderboard,matterbridge-bot,mina-staking-ledgers-exporter # The `./` makes it relative to the chart-search-root - --template-files=./README.md.gotmpl diff --git a/matterbridge-bot/README.md b/matterbridge-bot/README.md index 60370e18..17346895 100644 --- a/matterbridge-bot/README.md +++ b/matterbridge-bot/README.md @@ -63,3 +63,4 @@ helmfile status | tolerations | list | `[]` | Tolerations | | volumeMounts | list | `[]` | Additional volumeMounts on the output Deployment definition. | | volumes | list | `[]` | Additional volumes on the output Deployment definition. | + diff --git a/mina-staking-ledgers-exporter/README.md b/mina-staking-ledgers-exporter/README.md index 3b4b694f..66305fd6 100644 --- a/mina-staking-ledgers-exporter/README.md +++ b/mina-staking-ledgers-exporter/README.md @@ -1,72 +1,74 @@ -# Mina Staking Ledger Exporter +# mina-staking-ledgers-exporter -mina-staking-ledgers-exporter is a tools that generate Mina Staking Ledgers, and publish them on S3 and Mina Payout Data Provider API. +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) -## TL;DR - -```console -git clone https://github.com/MinaFoundation/helm-charts -cd helm-charts/mina-staking-ledgers-exporter -helm install RELEASE_NAME ./ --namespace NAMESPACE -``` +A Helm chart for Kubernetes ## Prerequisites -- Kubernetes 1.12+ -- Helm 3.1.0 +Before using this Helm chart, you should have the following prerequisites: + +- Access to Kubernetes cluster (If needed contact your friendly neighbourhood DevOps engineer) +- Helm >= v3.14.3 +- (**Optional**) helmfile >= v0.162.0 to install this chart -## Installing the Chart +## Installation -To install the chart with the release name `RELEASE_NAME`: +> Note: **examples** can be found in the repository -```console -helm install RELEASE_NAME ./ --namespace NAMESPACE +To install this Helm chart, the easiest is to create a helmfile.yaml with needed values and run: + +``` +helmfile template +helmfile apply ``` -The command deploys mina-staking-ledgers-exporter on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. +Or use helmfile only to generate resources and apply them with kubectl like so: -## Uninstalling the Chart +``` +helmfile template | kubectl -f - +``` + +Verify that the chart is deployed successfully: -To uninstall/delete the `RELEASE_NAME` deployment: +> Note: `kubectl` is a better suited tool for this -```console -helm delete RELEASE_NAME ``` +helmfile status +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | Affinity rules | +| fullnameOverride | string | `""` | The full release name override | +| image.pullPolicy | string | `"IfNotPresent"` | The pullPolicy used when pulling the image | +| image.repository | string | `"673156464838.dkr.ecr.us-west-2.amazonaws.com/github-actions-runner"` | The repository of the image | +| image.tag | string | `"default-2024.03"` | The tag of the image. Overrides the image tag whose default is the chart appVersion. | +| imagePullSecrets | list | `[]` | The secrets used to pull the image | +| minaStakingLedgersExporter.logLevel | string | `"info"` | The log level | +| minaStakingLedgersExporter.minaNodeLabel | string | `nil` | The Mina node label used to query the ledger | +| minaStakingLedgersExporter.minaPayoutsDataProvider.enable | bool | `true` | Enable upload to Mina payouts data provider | +| minaStakingLedgersExporter.minaPayoutsDataProvider.password | string | `nil` | The Mina payouts data provider password | +| minaStakingLedgersExporter.minaPayoutsDataProvider.url | string | `nil` | The Mina payouts data provider URL | +| minaStakingLedgersExporter.minaPayoutsDataProvider.username | string | `nil` | The Mina payouts data provider username | +| minaStakingLedgersExporter.network | string | `nil` | The network (mainnet | devnet) | +| minaStakingLedgersExporter.s3.bucket | string | `nil` | The S3 bucket | +| minaStakingLedgersExporter.s3.enable | bool | `true` | Enable upload to S3 | +| minaStakingLedgersExporter.s3.subpath | string | `nil` | The S3 subpath | +| minaStakingLedgersExporter.slackWebhookInfoUrl | string | `nil` | The Slack webhook URL for info messages | +| minaStakingLedgersExporter.slackWebhookWarnUrl | string | `nil` | The Slack webhook URL for warn messages | +| nameOverride | string | `""` | The release name override | +| nodeSelector | object | `{}` | Node selector labels | +| podAnnotations | object | `{}` | Annotations to add to the pods | +| podSecurityContext | object | `{}` | The Pod Security Context | +| resources | object | `{}` | Resource limitations for the pods | +| restartPolicy | string | `"Never"` | The restart policy | +| schedule | string | `"0 0 * * *"` | The cronjob schedule | +| securityContext | object | `{}` | The Security Context | +| serviceAccount.annotations | object | `{}` | Annotations to add to the service account | +| serviceAccount.create | bool | `true` | Specifies whether a service account should be created | +| serviceAccount.name | string | `""` | If not set and create is true, a name is generated using the fullname template | +| tolerations | list | `[]` | Tolerations | -The command removes all the Kubernetes components associated with the chart and deletes the release. - -## Parameters - -| Name | Description | Value | -| ----------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -------------------------------------------------------------------- | -| `image.repository` | mina-staking-ledgers-exporter image name | `673156464838.dkr.ecr.us-west-2.amazonaws.com/github-actions-runner` | -| `image.pullPolicy` | mina-staking-ledgers-exporter image pull policy | `IfNotPresent` | -| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` | -| `nameOverride` | String to partially override common.names.fullname | "" | -| `fullnameOverride` | String to fully override common.names.fullname | "" | -| `serviceAccount.create` | Enable the creation of a ServiceAccount for mina-staking-ledgers-exporter pods | `true` | -| `serviceAccount.annotations` | Annotations for the created ServiceAccount | {} | -| `serviceAccount.name` | Name of the created ServiceAccount | "" | -| `podAnnotations` | Annotations for mina-staking-ledgers-exporter pods | {} | -| `podLabels` | Extra labels for mina-staking-ledgers-exporter pods | {} | -| `podRegexPattern` | Regex pattern to match pods | ".*" | -| `schedule` | Schedule to run pod rotation, runs every 6 hours | "0 */6 * * *" | -| `restartPolicy` | Restart Policy when the job fails, can be OnFailure, Never, Always | "OnFailure" | -| `podSecurityContext` | Set mina-staking-ledgers-exporter Pod's Security Context | {} | -| `securityContext` | Set mina-staking-ledgers-exporter Security Context | {} | -| `resources.limits` | The resources limits for the mina-staking-ledgers-exporter container | {} | -| `resources.requests` | The resources requests for the mina-staking-ledgers-exporter container | {} | -| `nodeSelector` | Node labels for pod assignment | {} | -| `tolerations` | Tolerations for pod assignment | {} | -| `affinity` | Affinity for pod assignment | {} | -| `schedule` | Frequency to run the job | `0 0 * * *` | -| `restartPolicy` | Restart Policy | `Never` | -| `minaStakingLedgersExporter.network` | Network to run the Exporter against | ` ` | -| `minaStakingLedgersExporter.s3.bucket` | Bucket to upload the Mina Staking Ledgers | ` ` | -| `minaStakingLedgersExporter.s3.subpath` | Bucket subpath to upload the Mina Staking Ledgers | ` ` | -| `minaStakingLedgersExporter.minaNodeLabel` | Label of the Mina Daemon to execute Staking Ledger Generation | ` ` | -| `minaStakingLedgersExporter.slackWebhookInfoUrl` | Slack Webhook Info URL | ` ` | -| `minaStakingLedgersExporter.slackWebhookWarnUrl` | Slack Webhook Warn URL | ` ` | -| `minaStakingLedgersExporter.minaPayoutsDataProvider.url` | Mina Payouts Data Provider URL | ` ` | -| `minaStakingLedgersExporter.minaPayoutsDataProvider.username` | Mina Payouts Data Provider Username | ` ` | -| `minaStakingLedgersExporter.minaPayoutsDataProvider.password` | Mina Payouts Data Provider Password | ` ` | diff --git a/mina-staking-ledgers-exporter/scripts/mina-staking-ledgers-exporter.py b/mina-staking-ledgers-exporter/scripts/mina-staking-ledgers-exporter.py index 0ccfe0dd..8ab47e57 100755 --- a/mina-staking-ledgers-exporter/scripts/mina-staking-ledgers-exporter.py +++ b/mina-staking-ledgers-exporter/scripts/mina-staking-ledgers-exporter.py @@ -15,12 +15,13 @@ class config: - + api_enable = os.environ.get("APP_MINA_PAYOUTS_DATA_PROVIDER_ENABLE", False).lower() == "true" api_password = os.environ["APP_MINA_PAYOUTS_DATA_PROVIDER_PASSWORD"] api_url = os.environ["APP_MINA_PAYOUTS_DATA_PROVIDER_URL"] api_username = os.environ["APP_MINA_PAYOUTS_DATA_PROVIDER_USERNAME"] mina_node_label = os.environ["APP_MINA_NODE_LABEL"] network = os.environ["APP_NETWORK"] + s3_enable = os.environ.get("APP_S3_ENABLE", False).lower() == "true" s3_bucket = os.environ["APP_S3_BUCKET"] s3_subpath = os.environ.get("APP_S3_SUBPATH", "") slack_webhook_info_url = os.environ["APP_SLACK_WEBHOOK_INFO_URL"] @@ -79,16 +80,24 @@ def process_staking_ledger(synced_pod, epoch, staking_ledger): tar.extractall(path="./") validate_json_file(staking_ledger_name) - logger.info("Uploading to S3") - uploaded_to_s3 = upload_file_to_s3( - staking_ledger_archive_name, - config.s3_bucket, - config.s3_subpath, - staking_ledger_archive_name, - ) - - logger.info("Uploading to Payout API") - uploaded_to_api = submit_mina_payout_data(staking_ledger_name, staking_ledger_hash, epoch) + if config.s3_enable: + logger.info("Uploading to S3") + uploaded_to_s3 = upload_file_to_s3( + staking_ledger_archive_name, + config.s3_bucket, + config.s3_subpath, + staking_ledger_archive_name, + ) + else: + logger.info("S3 upload is disabled, skip") + uploaded_to_s3 = False + + if config.api_enable: + logger.info("Uploading to Payout API") + uploaded_to_api = submit_mina_payout_data(staking_ledger_name, staking_ledger_hash, epoch) + else: + logger.info("API upload is disabled, skip") + uploaded_to_api = False logger.info("Sending notifications") message = "" diff --git a/mina-staking-ledgers-exporter/templates/cronjob.yaml b/mina-staking-ledgers-exporter/templates/cronjob.yaml index 65d81d81..32e686d3 100644 --- a/mina-staking-ledgers-exporter/templates/cronjob.yaml +++ b/mina-staking-ledgers-exporter/templates/cronjob.yaml @@ -36,12 +36,16 @@ spec: env: - name: APP_NETWORK value: "{{ .Values.minaStakingLedgersExporter.network }}" + - name: APP_S3_ENABLE + value: "{{ .Values.minaStakingLedgersExporter.s3.enable }}" - name: APP_S3_BUCKET value: "{{ .Values.minaStakingLedgersExporter.s3.bucket }}" - name: APP_S3_SUBPATH value: "{{ .Values.minaStakingLedgersExporter.s3.subpath }}" - name: APP_MINA_NODE_LABEL value: "{{ .Values.minaStakingLedgersExporter.minaNodeLabel }}" + - name: APP_MINA_PAYOUTS_DATA_PROVIDER_ENABLE + value: "{{ .Values.minaStakingLedgersExporter.minaPayoutsDataProvider.enable }}" - name: APP_MINA_PAYOUTS_DATA_PROVIDER_URL value: "{{ .Values.minaStakingLedgersExporter.minaPayoutsDataProvider.url }}" - name: APP_MINA_PAYOUTS_DATA_PROVIDER_USERNAME diff --git a/mina-staking-ledgers-exporter/values.yaml b/mina-staking-ledgers-exporter/values.yaml index 5fb3d310..88e5d2a1 100644 --- a/mina-staking-ledgers-exporter/values.yaml +++ b/mina-staking-ledgers-exporter/values.yaml @@ -3,46 +3,70 @@ # Declare variables to be passed into your templates. image: + # -- The repository of the image repository: 673156464838.dkr.ecr.us-west-2.amazonaws.com/github-actions-runner + # -- The pullPolicy used when pulling the image pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. + # -- The tag of the image. Overrides the image tag whose default is the chart appVersion. tag: "default-2024.03" +# -- The secrets used to pull the image imagePullSecrets: [] +# -- The release name override nameOverride: "" +# -- The full release name override fullnameOverride: "" +# -- The cronjob schedule schedule: "0 0 * * *" +# -- The restart policy restartPolicy: "Never" minaStakingLedgersExporter: + # -- The log level logLevel: "info" + # -- The network (mainnet | devnet) network: s3: + # -- Enable upload to S3 + enable: true + # -- The S3 bucket bucket: + # -- The S3 subpath subpath: + # -- The Mina node label used to query the ledger minaNodeLabel: + # -- The Slack webhook URL for info messages slackWebhookInfoUrl: + # -- The Slack webhook URL for warn messages slackWebhookWarnUrl: minaPayoutsDataProvider: + # -- Enable upload to Mina payouts data provider + enable: true + # -- The Mina payouts data provider URL url: + # -- The Mina payouts data provider username username: + # -- The Mina payouts data provider password password: serviceAccount: - # Specifies whether a service account should be created + # -- Specifies whether a service account should be created create: true - # Annotations to add to the service account + # -- Annotations to add to the service account annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template + # -- The name of the service account to use. + # -- If not set and create is true, a name is generated using the fullname template name: "" +# -- Annotations to add to the pods podAnnotations: {} +# -- The Pod Security Context podSecurityContext: {} # fsGroup: 2000 +# -- The Security Context securityContext: {} # capabilities: # drop: @@ -51,7 +75,7 @@ securityContext: {} # runAsNonRoot: true # runAsUser: 1000 - +# -- Resource limitations for the pods resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -64,8 +88,11 @@ resources: {} # cpu: 100m # memory: 128Mi +# -- Node selector labels nodeSelector: {} +# -- Tolerations tolerations: [] +# -- Affinity rules affinity: {}