diff --git a/.github/workflows/evaluation.yml b/.github/workflows/evaluation.yml deleted file mode 100644 index 6870b5d2..00000000 --- a/.github/workflows/evaluation.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Evaluation Matrix -on: - push: - branches: - - main - -permissions: - deployments: write - contents: write - -jobs: - evaluation: - name: Performance Evaluation - runs-on: ubuntu-latest - strategy: - matrix: - evaluation: - - provider: ChatIBMGenAI - model_prefix: codellama - model: codellama-34b-instruct - - provider: ChatIBMGenAI - model_prefix: deepseek-ai - model: deepseek-coder-33b-instruct - - provider: ChatIBMGenAI - model_prefix: meta-llama - model: llama-3-70b-instruct - max_new_tokens: 2048 - - provider: ChatIBMGenAI - model_prefix: mistralai - model: mistral-7b-v0-1 - - provider: ChatIBMGenAI - model_prefix: mistralai - model: mixtral-8x7b-instruct-v01 - test: - - example: example_a - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@main - with: - python-version: 3.12 - - name: Update environment - run: | - echo "MAX_NEW_TOKENS=${{ matrix.evaluation.max_new_tokens }}" >> $GITHUB_ENV - if: matrix.evaluation.max_new_tokens != null - - name: Run benchmark - run: | - pip install -r requirements.txt - pip install -e . - cd kai - cat << EOF > config.toml - log_level = "info" - file_log_level = "debug" - log_dir = "$pwd/logs" - demo_mode = false - trace_enabled = true - solution_consumers = ["diff_only", "llm_summary"] - [incident_store] - solution_detectors = "naive" - solution_producers = "text_only" - [incident_store.args] - provider = "postgresql" - host = "127.0.0.1" - database = "kai" - user = "kai" - password = "dog8code" - [embeddings] - todo = true - [models] - provider = "${{ matrix.evaluation.provider }}" - [models.args] - model_id = "${{ matrix.evaluation.model_prefix }}/${{ matrix.evaluation.model }}" - EOF - if [[ ! -z "${MAX_NEW_TOKENS}" ]]; then - cat << EOF >> config.toml - parameters.max_new_tokens = ${{ matrix.evaluation.max_new_tokens }} - EOF - fi - echo [{\"name\": \ - \"${{ matrix.evaluation.provider }}_${{ matrix.evaluation.model_prefix }}_${{ matrix.evaluation.model }}_${{ matrix.test.example }}\", \ - \"unit\": \"Match\", \ - \"value\": \"$(PYTHONPATH=".." python evaluation.py --configs ./config.toml | tail -n 1 | awk '{ print $3 }')\" \ - }] > ../output.txt - git checkout config.toml - cd .. - env: - GENAI_KEY: ${{ secrets.GENAI_KEY }} - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: customBiggerIsBetter - benchmark-data-dir-path: evaluations - output-file-path: output.txt - #fail-on-alert: true - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true diff --git a/.github/workflows/march-image-build-push.yml b/.github/workflows/march-image-build-push.yml deleted file mode 100644 index 11bd9ba9..00000000 --- a/.github/workflows/march-image-build-push.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Build and Push Multi-Arch Image" - -on: - workflow_dispatch: - push: - branches: - - "main" - - "demo.*" - tags: - - "stable" - - "v*" - - "demo.*" - -concurrency: - group: march-build-${{ github.ref }} - cancel-in-progress: true - -jobs: - push-quay: - name: Build and Push Manifest - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - steps: - - name: Build Container - uses: konveyor/release-tools/build-push-quay@main - with: - architectures: "amd64, arm64" - containerfile: ./build/Containerfile - extra-args: | - --storage-opt=overlay.mount_program= - image_name: kai - image_namespace: konveyor - image_registry: quay.io - quay_publish_robot: ${{ secrets.QUAY_PUBLISH_ROBOT }} - quay_publish_token: ${{ secrets.QUAY_PUBLISH_TOKEN }} - ref: ${{ github.ref }}