From 72c7a261ed249f7858a606c5ef5dd5f7213b8539 Mon Sep 17 00:00:00 2001 From: Wenxin Zhang Date: Sat, 11 May 2024 16:46:09 +0800 Subject: [PATCH] update ocveragerc Signed-off-by: Wenxin Zhang --- .github/workflows/model_test.yml | 99 +++++++++++++++++++ .../workflows/scripts/models/collect_log.sh | 34 +++++++ .../workflows/scripts/models/model_test.sh | 64 ++++++++++++ .github/workflows/scripts/unittest/coveragerc | 8 -- .../workflows/scripts/unittest/unittest.sh | 4 +- .github/workflows/unittest.yml | 7 -- 6 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/model_test.yml create mode 100644 .github/workflows/scripts/models/collect_log.sh create mode 100644 .github/workflows/scripts/models/model_test.sh diff --git a/.github/workflows/model_test.yml b/.github/workflows/model_test.yml new file mode 100644 index 00000000..7d39459c --- /dev/null +++ b/.github/workflows/model_test.yml @@ -0,0 +1,99 @@ +name: Model Test + +on: + workflow_dispatch: + +# If there is a new commit, the previous jobs will be canceled +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +permissions: write-all +env: + OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/scripts/models + SCRIPT_PATH: /GenAIEval/.github/workflows/scripts + REPO_NAME: "GenAIEval" + DOCKER_TAG: "latest" + DOCKER_FILE_NAME: "model.dockerfile" + CONTAINER_NAME: "modelTest" + + +jobs: + Evaluation-Workflow: + runs-on: aise-cluster + strategy: + matrix: + include: + - modelName: "EleutherAI/gpt-j-6B" + task: "hellaswag" + device: "cpu" + fail-fast: true + + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + submodules: "recursive" + fetch-tags: true + # We need this because GitHub needs to clone the branch to pipeline + - name: Docker Build + run: | + docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }} -t ${{ env.REPO_NAME }}:${{ env.DOCKER_TAG }} . + + - name: Docker Run + run: | + if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then + docker stop ${{ env.CONTAINER_NAME }} + docker rm -vf ${{ env.CONTAINER_NAME }} || true + fi + docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \ + -v ${{ github.workspace }}:/GenAIEval \ + ${{ env.REPO_NAME }}:${{ env.DOCKER_TAG }} + + - name: Binary build + run: | + docker exec ${{ env.CONTAINER_NAME }} \ + bash -c "cd /GenAIEval && pip install -r requirements.txt && python setup.py install" + + #- name: Download Reference Artifact + # id: download-artifact + # uses: dawidd6/action-download-artifact@v3.1.2 + # with: + # workflow: model_test.yml + # name: ${{ matrix.device }}-${{ matrix.modelName }} + # run_id: ${{ vars.ModelTest_REF_ID }} + # path: ${{ github.workspace }}/${{ matrix.device }}_${{ matrix.modelName }}_refer_log + # name_is_regexp: true + # repo: ${{ github.repository }} + # check_artifacts: false + # search_artifacts: false + # skip_unpack: false + # if_no_artifact_found: warn + + #- name: Display structure of downloaded files + # run: ls -R + + - name: Evaluation + run: | + docker exec ${{ env.CONTAINER_NAME }} \ + bash -c "cd /GenAIEval/.github/workflows/scripts/models \ + && bash model_test.sh --model=${{ matrix.modelName }} --device=${{ matrix.device }} --tasks=${{ matrix.task }}" + + - name: Collect Log + run: | + docker exec ${{ env.CONTAINER_NAME }} \ + bash -c "cd /GenAIEval/.github/workflows/scripts/models \ + && bash -x collect_log.sh --model=${{ matrix.modelName }} \ + --device=${{ matrix.device }} \ + --task=${{ matrix.task }} + + - name: Publish pipeline artifact + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: ${{ matrix.device }}-${{ matrix.modelName }} + path: ${{ github.workspace }}/${{ matrix.device }}/${{ matrix.modelName }} + if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn` + retention-days: 60 # 1 <= retention-days <= 90 \ No newline at end of file diff --git a/.github/workflows/scripts/models/collect_log.sh b/.github/workflows/scripts/models/collect_log.sh new file mode 100644 index 00000000..4b14eaa0 --- /dev/null +++ b/.github/workflows/scripts/models/collect_log.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -eo pipefail +source /GenAIEval/.github/workflows/script/change_color.sh + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' +PERF_STABLE_CHECK=true +for i in "$@"; do + case $i in + --device=*) + device=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --task=*) + task=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +output_file="/GenAIEval/${device}/${model}/${device}-${model}-${task}.log" +$BOLD_YELLOW && echo "-------- Collect logs --------" && $RESET + +echo "working in" +pwd +if [[ ! -f ${output_file} ]]; then + echo "${framework},${mode},${model},batch,cores per instance,Troughput,${precision}," >> ${WORKSPACE}/summary.log +else + throughput=$(grep -A 2 "best," ${output_file} | tail -1 | awk -F "," '{print $1","$3","$4}') + log_file=$(grep -A 2 "best," ${output_file} | tail -1 | awk -F "=" '{print $NF}' | awk '{print $1}' | sed 's|\"||g') + logfile=${log_file##*/} + logfile=${logs_prefix_url}${logfile} + echo "${framework},${mode},${model},${throughput},${precision},${logfile}" >> ${WORKSPACE}/summary.log +fi \ No newline at end of file diff --git a/.github/workflows/scripts/models/model_test.sh b/.github/workflows/scripts/models/model_test.sh new file mode 100644 index 00000000..459659de --- /dev/null +++ b/.github/workflows/scripts/models/model_test.sh @@ -0,0 +1,64 @@ +#!/bin/bash +set -eo pipefail +source /GenAIEval/.github/workflows/script/change_color.sh + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' +PERF_STABLE_CHECK=true +for i in "$@"; do + case $i in + --device=*) + device=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --task=*) + task=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +log_dir="/GenAIEval/${device}/${model}" +mkdir -p ${log_dir} + +$BOLD_YELLOW && echo "-------- evaluation start --------" && $RESET + +main() { + #prepare + run_benchmark +} + +function prepare() { + ## prepare env + working_dir="/GenAIEval" + cd ${working_dir} + echo "Working in ${working_dir}" + echo -e "\nInstalling model requirements..." + if [ -f "requirements.txt" ]; then + python -m pip install -r requirements.txt + pip list + else + echo "Not found requirements.txt file." + fi +} + +function run_benchmark() { + cd ${working_dir} + pip install --upgrade-strategy eager optimum[habana] + overall_log="${log_dir}/${device}-${model}-${task}.log" + python main.py \ + --model hf \ + --model_args pretrained=${model} \ + --tasks ${task} \ + --device ${device} \ + --batch_size 8 + 2>&1 | tee ${overall_log} + + status=$? + if [ ${status} != 0 ]; then + echo "Evaluation process returned non-zero exit code." + exit 1 + fi +} + +main diff --git a/.github/workflows/scripts/unittest/coveragerc b/.github/workflows/scripts/unittest/coveragerc index 77fcdf08..f2d68ae8 100644 --- a/.github/workflows/scripts/unittest/coveragerc +++ b/.github/workflows/scripts/unittest/coveragerc @@ -7,11 +7,3 @@ omit = */**/fake.py exclude_lines = pragma: no cover - raise NotImplementedError - raise TypeError - if self.device == "gpu": - if device == "gpu": - except ImportError: - except Exception as e: - onnx_version < ONNX18_VERSION - onnx_version >= ONNX18_VERSION diff --git a/.github/workflows/scripts/unittest/unittest.sh b/.github/workflows/scripts/unittest/unittest.sh index faf96c41..a241c471 100644 --- a/.github/workflows/scripts/unittest/unittest.sh +++ b/.github/workflows/scripts/unittest/unittest.sh @@ -15,9 +15,9 @@ # limitations under the License. #!/bin/bash -source /GenAIEval/.github/workflows/scripts/change_color.sh +source /GenAIEval/.github/workflows/scripts/change_color export COVERAGE_RCFILE="/GenAIEval/.github/workflows/scripts/unittest/coveragerc" -LOG_DIR=/log_dir +LOG_DIR=/GenAIEval/log_dir mkdir -p ${LOG_DIR} # get parameters PATTERN='[-a-zA-Z0-9_]*=' diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index f7da02c2..8562165d 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -82,13 +82,6 @@ jobs: docker exec ${{ env.CONTAINER_NAME }} \ bash -c "bash /GenAIEval/.github/workflows/scripts/unittest/unittest.sh --test_name=${{ matrix.test_name }}" - - name: Collect log - if: ${{ !cancelled() }} - run: | - docker exec ${{ env.CONTAINER_NAME }} \ - bash -c "cd /GenAIEval && \ - mv /log_dir . " - - name: Publish pipeline artifact uses: actions/upload-artifact@v4 if: ${{ !cancelled() }}