Updates and refactors

Signed-off-by: Artem Kladov <artem.kladov@flant.com>

.github/workflow_templates/validation.yml

@@ -124,11 +124,11 @@ jobs:
 {!{ tmpl.Exec "validation_template" (slice $ctx "no_cyrillic") | strings.Indent 4 }!}
 
   doc_validation:
-    name: Documentation Validation
+    name: Docs Validation (base)
 {!{ tmpl.Exec "validation_template" (slice $ctx "doc_changes") | strings.Indent 4 }!}
 
   doc_spell_check:
-    name: Documentation spell checking
+    name: Docs Validation (spellcheck)
 {!{ tmpl.Exec "spellcheck_template" (slice $ctx) | strings.Indent 4 }!}
 
   copyright_validation:
@@ -219,10 +219,6 @@ steps:
 
   - name: Spell check
     run: |
-      cat ./pr.diff | grep -E "^(---|\+\+\+) [ab]" | cut -d \/ -f 2- | sort -u > tools/spelling/internal/pr.diff.files
-      cd tools/spelling
-      pwd
-      ls -la
-      werf run docs-spell-checker --dev --docker-options="--entrypoint=sh" -- /app/internal/container_spell_check.sh --list /app/internal/pr.diff.files --skipmissing
+      make lint-doc-spellcheck-pr
    # </template: spellcheck_template>
 {!{- end -}!}

.github/workflows/validation.yml

@@ -386,7 +386,7 @@ jobs:
           ./.github/scripts/validation_run.sh ./testing/validate_no_cyrillic.sh
 
   doc_validation:
-    name: Documentation Validation
+    name: Docs Validation (base)
 
     needs:
       - discover
@@ -415,7 +415,7 @@ jobs:
           ./.github/scripts/validation_run.sh ./testing/validate_doc_changes.sh
 
   doc_spell_check:
-    name: Documentation spell checking
+    name: Docs Validation (spellcheck)
     needs:
       - discover
       - pull_request_info
@@ -441,11 +441,7 @@ jobs:
 
       - name: Spell check
         run: |
-          cat ./pr.diff | grep -E "^(---|\+\+\+) [ab]" | cut -d \/ -f 2- | sort -u > tools/spelling/internal/pr.diff.files
-          cd tools/spelling
-          pwd
-          ls -la
-          werf run docs-spell-checker --dev --docker-options="--entrypoint=sh" -- /app/internal/container_spell_check.sh --list /app/internal/pr.diff.files --skipmissing
+          make lint-doc-spellcheck-pr
        # </template: spellcheck_template>
 
   copyright_validation:

Makefile

@@ -245,7 +245,10 @@ docs-down: ## Stop all the documentation containers (e.g. site_site_1 - for Linu
 .PHONY: docs-spellcheck
 docs-spellcheck: ## Check the spelling in the documentation.
   ##~ Options: file="path/to/file" (Specify a path to a specific file)
-	cd tools/spelling && sh spell_check.sh -f $(filename)
+	cd tools/spelling && werf run docs-spell-checker --dev --docker-options="--entrypoint=sh" -- /app/spell_check.sh -f $(file)
+
+lint-doc-spellcheck-pr:
+	@cd tools/spelling && werf run docs-spell-checker --dev --docker-options="--entrypoint=bash" -- /app/check_diff.sh
 
 .PHONY: docs-spellcheck-generate-dictionary
 docs-spellcheck-generate-dictionary: ## Generate a dictionary (run it after adding new words to the tools/spelling/wordlist file).
@@ -257,10 +260,11 @@ docs-spellcheck-generate-dictionary: ## Generate a dictionary (run it after addi
 	@cat ./tools/spelling/wordlist | wc -l | sed 's/^[ \t]*//g' > ./tools/spelling/dictionaries/dev_OPS.dic
 	@sort ./tools/spelling/wordlist >> ./tools/spelling/dictionaries/dev_OPS.dic
 	@echo "Don't forget to commit changes and push it!"
+	@git diff --stat
 
 .PHONY: docs-spellcheck-get-typos-list
 docs-spellcheck-get-typos-list: ## Print out a list of all the terms in all pages that were considered as a typo.
-	cd tools/spelling && sh spell_check.sh | sed "1,/Checking/ d" | sed "/Checking/d" | sort -u
+	@cd tools/spelling && werf run docs-spell-checker --dev --docker-options="--entrypoint=sh" -- "/app/spell_check.sh" 2>/dev/null | sed "1,/Spell check the documentation/ d; /^Possible typos/d" | sort -u
 
 ##@ Update kubernetes control-plane patchversions
 

tools/spelling/check_diff.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+. $(dirname "$(realpath "$0")")/spell_lib.sh
+
+diff_start_re='^diff --git a/(.*) b/(.*)$'
+new_file_name_re='^\+\+\+ (/dev/null|b/(.*))$'
+end_metadata_re='^@@[- 0-9,+]+@@(.*)$'
+
+first_line=true
+metadata_block=false
+skip_file=false
+pr_diff=./pr.diff
+
+file_name=""
+file_changes=""
+
+exit_code=0
+
+function check() {
+  local file_name=$1
+  local file_changes=$2
+
+  if [[ -n "${file_name}" ]] && [[ -n "${file_changes}" ]]; then
+    local result=$(file_diff_check_spell ${file_name} "${file_changes}")
+    if [ -n "${result}" ]; then
+      print_message_about_typos_in_a_file "${file_name}"
+      file_name
+      echo "${result}"
+      exit_code=1
+    fi
+  fi
+}
+
+while IFS= read -r line
+do
+  if [[ "${line}" =~ ${diff_start_re} ]]; then
+    check "${file_name}" "${file_changes}"
+    metadata_block=true
+    skip_file=false
+    file_changes=""
+    continue
+  elif [[ "${line}" =~ $new_file_name_re ]]; then
+    file_name=${BASH_REMATCH[2]}
+    skip_file=false
+    if ! validate_file_name "${file_name}"; then
+      skip_file=true
+      continue
+    fi
+    continue
+  elif [[ "${line}" =~ $end_metadata_re ]]; then
+    metadata_block=false
+    continue
+  fi
+  if [ ${metadata_block} = true ] || [ ${skip_file} = true ]; then
+    continue
+  fi
+  file_changes+=$(echo "${line}" | sed 's/^[+-]//')
+done < ${pr_diff}
+
+check "${file_name}" "${file_changes}"
+
+exit ${exit_code}

tools/spelling/internal/clean-files.py → tools/spelling/clean-files.py

@@ -49,6 +49,17 @@ def find_all_keys(input_dict: dict) -> list:
     return result
 
 if len (sys.argv) > 1:
+  if sys.argv[1] == '-':
+    text = sys.stdin.read()
+    text = clean_preamble(text)
+    text = delete_code_blocks(text)
+    text = delete_md_links(text)
+    text = clean_scripts(text)
+    text = clean_html(text)
+    text = clean_liquid(text)
+    text = delete_nbsp(text)
+    print(text)
+  else:
     file_extension = sys.argv[1].split('.')[-1]
     if file_extension == 'html' or file_extension == 'md' or file_extension == 'liquid':
         with open(sys.argv[1], 'r') as f:

tools/spelling/dictionaries/dev_OPS.dic

@@ -1,4 +1,4 @@
-1851
+1853
 1GB
 2GB
 5XX
@@ -177,6 +177,7 @@ CNCF
 CNCF’s
 cni
 CNI
+CoC
 codec
 Colour
 comparations
@@ -715,6 +716,7 @@ NOERROR
 NoExecute
 NogeGroup
 nonreplicated
+NoOps
 noProxyAddressList
 NoSchedule
 NoSQL

tools/spelling/spell_check.sh

@@ -2,4 +2,76 @@
 
 set -e
 
-werf run docs-spell-checker --dev --docker-options="--entrypoint=sh" -- /app/internal/container_spell_check.sh $@
+. $(dirname "$(realpath "$0")")/spell_lib.sh
+
+SKIP_MISSING_FILES=0
+exit_code=0
+
+HELP_STRING=$(cat <<EOF
+Usage: spell_check.sh [OPTION]
+
+Optional arguments:
+  -f PATH, --file PATH         the name of the file with a path (relative from the Deckhouse repo)
+  -l PATH, --list PATH         the name of the file with a list of files to check (relative from the Deckhouse repo)
+  --skipmissing                skip missing files
+  -h, --help         output this message
+EOF
+)
+
+while true; do
+  case "$1" in
+    -f | --file )
+      if [ -n "$2" ]; then
+        if [ -f "$2" ]; then
+          file_name=$2; shift 2
+        else
+          echo "File $2 not found" >&2
+          exit 1
+        fi
+      else
+        shift 1
+      fi
+      ;;
+    -l | --list )
+      files_list_filename=$2; shift 2;;
+    --skipmissing )
+      SKIP_MISSING_FILES=1; shift 1;;
+    -h | --help )
+      echo "$HELP_STRING"; exit 0 ;;
+    * )
+      break ;;
+  esac
+done
+
+echo "Spell check the documentation..."
+
+if [ -n "${file_name}" ]; then
+    if validate_file_name "${file_name}"; then
+        print_message_about_typos_in_a_file "${file_name}"
+        result="$(file_check_spell ${file_name})"
+        if [ -n "$result" ]; then
+          echo $result | sed 's/\s\+/\n/g'
+          exit_code=1
+        fi
+    fi
+else
+  if [ -n "${files_list_filename}" ]; then
+      files_search_path="$(cat ${files_list_filename})"
+  else
+      files_search_path="$(find ./ -type f)"
+  fi
+  for file_name in ${files_search_path};
+  do
+    if validate_file_name "${file_name}"; then
+        result="$(file_check_spell "${file_name}")"
+        if [ -n "$result" ]; then
+          exit_code=1
+          print_message_about_typos_in_a_file "${file_name}"
+          echo ${result} | sed 's/\s\+/\n/g'
+          echo
+        fi
+    fi
+  done
+fi
+
+exit ${exit_code}