From ba1f5149042b5d4facb297a2b3db49501a36db64 Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Tue, 28 Jan 2025 14:17:03 +0000 Subject: [PATCH] Simplified logic for migration complexity explanation summary in html report - moved the html template code for summary to the parent template file - passing combined struct(report + explanation summary) to the html template --- yb-voyager/cmd/assessMigrationCommand.go | 49 +++------ yb-voyager/cmd/common.go | 2 +- yb-voyager/cmd/migration_complexity.go | 103 ++---------------- .../migration_assessment_report.template | 40 ++++++- yb-voyager/src/callhome/diagnostics.go | 2 +- 5 files changed, 69 insertions(+), 127 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index d43c4c2c23..6daddc607d 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -162,13 +162,6 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { }), } - // we will build this twice for json and html reports both, at the time of report generation. - // whatever happens later will be stored in the struct's field. so to be on safer side, we will build it again here as per required format. - explanation, err := buildMigrationComplexityExplanation(source.DBType, assessmentReport, "") - if err != nil { - log.Errorf("failed to build migration complexity explanation for callhome assessment payload: %v", err) - } - var obfuscatedIssues []callhome.AssessmentIssueCallhome for _, issue := range assessmentReport.Issues { obfuscatedIssue := callhome.AssessmentIssueCallhome{ @@ -195,7 +188,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { PayloadVersion: callhome.ASSESS_MIGRATION_CALLHOME_PAYLOAD_VERSION, TargetDBVersion: assessmentReport.TargetDBVersion, MigrationComplexity: assessmentReport.MigrationComplexity, - MigrationComplexityExplanation: explanation, + MigrationComplexityExplanation: assessmentReport.MigrationComplexityExplanation, SchemaSummary: callhome.MarshalledJsonString(schemaSummaryCopy), Issues: callhome.MarshalledJsonString(obfuscatedIssues), Error: callhome.SanitizeErrorMsg(errMsg), @@ -206,7 +199,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { } payload.PhasePayload = callhome.MarshalledJsonString(assessPayload) - err = callhome.SendPayload(&payload) + err := callhome.SendPayload(&payload) if err == nil && (status == COMPLETE || status == ERROR) { callHomeErrorOrCompletePayloadSent = true } @@ -463,19 +456,13 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE } assessmentIssues := convertAssessmentIssueToYugabyteDAssessmentIssue(assessmentReport) - // we will build this twice for json and html reports both, at the time of report generation. - // whatever happens later will be stored in the struct's field. so to be on safer side, we will build it again here as per required format. - explanation, err := buildMigrationComplexityExplanation(source.DBType, assessmentReport, "") - if err != nil { - log.Errorf("failed to build migration complexity explanation for yugabyted assessment payload: %v", err) - } payload := AssessMigrationPayload{ PayloadVersion: ASSESS_MIGRATION_YBD_PAYLOAD_VERSION, VoyagerVersion: assessmentReport.VoyagerVersion, TargetDBVersion: assessmentReport.TargetDBVersion, MigrationComplexity: assessmentReport.MigrationComplexity, - MigrationComplexityExplanation: explanation, + MigrationComplexityExplanation: assessmentReport.MigrationComplexityExplanation, SchemaSummary: assessmentReport.SchemaSummary, AssessmentIssues: assessmentIssues, SourceSizeDetails: SourceDBSizeDetails{ @@ -828,7 +815,10 @@ func generateAssessmentReport() (err error) { addMigrationCaveatsToAssessmentReport(unsupportedDataTypesForLiveMigration, unsupportedDataTypesForLiveMigrationWithFForFB) // calculating migration complexity after collecting all assessment issues - assessmentReport.MigrationComplexity = calculateMigrationComplexity(source.DBType, schemaDir, assessmentReport) + complexity, explanation := calculateMigrationComplexityAndExplanation(source.DBType, schemaDir, assessmentReport) + log.Infof("migration complexity: %q and explanation: %q", complexity, explanation) + assessmentReport.MigrationComplexity = complexity + assessmentReport.MigrationComplexityExplanation = explanation assessmentReport.Sizing = migassessment.SizingReport assessmentReport.TableIndexStats, err = assessmentDB.FetchAllStats() @@ -1565,13 +1555,6 @@ func generateAssessmentReportJson(reportDir string) error { jsonReportFilePath := filepath.Join(reportDir, fmt.Sprintf("%s%s", ASSESSMENT_FILE_NAME, JSON_EXTENSION)) log.Infof("writing assessment report to file: %s", jsonReportFilePath) - var err error - assessmentReport.MigrationComplexityExplanation, err = buildMigrationComplexityExplanation(source.DBType, assessmentReport, "") - if err != nil { - return fmt.Errorf("unable to build migration complexity explanation for json report: %w", err) - } - log.Infof("migration complexity explanation: %q", assessmentReport.MigrationComplexityExplanation) - strReport, err := json.MarshalIndent(assessmentReport, "", "\t") if err != nil { return fmt.Errorf("failed to marshal the assessment report: %w", err) @@ -1590,12 +1573,6 @@ func generateAssessmentReportHtml(reportDir string) error { htmlReportFilePath := filepath.Join(reportDir, fmt.Sprintf("%s%s", ASSESSMENT_FILE_NAME, HTML_EXTENSION)) log.Infof("writing assessment report to file: %s", htmlReportFilePath) - var err error - assessmentReport.MigrationComplexityExplanation, err = buildMigrationComplexityExplanation(source.DBType, assessmentReport, "html") - if err != nil { - return fmt.Errorf("unable to build migration complexity explanation for html report: %w", err) - } - file, err := os.Create(htmlReportFilePath) if err != nil { return fmt.Errorf("failed to create file for %q: %w", filepath.Base(htmlReportFilePath), err) @@ -1625,7 +1602,17 @@ func generateAssessmentReportHtml(reportDir string) error { // marking this as empty to not display this in html report for PG assessmentReport.SchemaSummary.SchemaNames = []string{} } - err = tmpl.Execute(file, assessmentReport) + + type CombinedStruct struct { + AssessmentReport + MigrationComplexityCategorySummary []MigrationComplexityCategorySummary + } + combined := CombinedStruct{ + AssessmentReport: assessmentReport, + MigrationComplexityCategorySummary: buildCategorySummary(source.DBType, assessmentReport.Issues), + } + + err = tmpl.Execute(file, combined) if err != nil { return fmt.Errorf("failed to render the assessment report: %w", err) } diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 364ed7a6d6..83df68ea8e 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1158,7 +1158,7 @@ type AssessmentIssueYugabyteD struct { Type string `json:"Type"` // Ex: GIN_INDEXES, SECURITY_INVOKER_VIEWS, STORED_GENERATED_COLUMNS Name string `json:"Name"` // Ex: GIN Indexes, Security Invoker Views, Stored Generated Columns Description string `json:"Description"` // description based on type/name - Impact string `json:"Impact"` // // Level-1, Level-2, Level-3 (no default: need to be assigned for each issue) + Impact string `json:"Impact"` // Level-1, Level-2, Level-3 (no default: need to be assigned for each issue) ObjectType string `json:"ObjectType"` // For datatype category, ObjectType will be datatype (for eg "geometry") ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC) SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC) diff --git a/yb-voyager/cmd/migration_complexity.go b/yb-voyager/cmd/migration_complexity.go index a51ba282a8..1f01353d39 100644 --- a/yb-voyager/cmd/migration_complexity.go +++ b/yb-voyager/cmd/migration_complexity.go @@ -16,14 +16,12 @@ limitations under the License. package cmd import ( - "bytes" "encoding/csv" "fmt" "math" "os" "path/filepath" "strings" - "text/template" "github.com/samber/lo" log "github.com/sirupsen/logrus" @@ -45,9 +43,9 @@ var ( ) // Migration complexity calculation based on the detected assessment issues -func calculateMigrationComplexity(sourceDBType string, schemaDirectory string, assessmentReport AssessmentReport) string { +func calculateMigrationComplexityAndExplanation(sourceDBType string, schemaDirectory string, assessmentReport AssessmentReport) (string, string) { if sourceDBType != ORACLE && sourceDBType != POSTGRESQL { - return NOT_AVAILABLE + return NOT_AVAILABLE, "" } log.Infof("calculating migration complexity for %s...", sourceDBType) @@ -56,19 +54,19 @@ func calculateMigrationComplexity(sourceDBType string, schemaDirectory string, a migrationComplexity, err := calculateMigrationComplexityForOracle(schemaDirectory) if err != nil { log.Errorf("failed to get migration complexity for oracle: %v", err) - return NOT_AVAILABLE + return NOT_AVAILABLE, "" } - return migrationComplexity + return migrationComplexity, "" case POSTGRESQL: - return calculateMigrationComplexityForPG(assessmentReport) + return calculateMigrationComplexityAndExplanationForPG(assessmentReport) default: panic(fmt.Sprintf("unsupported source db type '%s' for migration complexity", sourceDBType)) } } -func calculateMigrationComplexityForPG(assessmentReport AssessmentReport) string { +func calculateMigrationComplexityAndExplanationForPG(assessmentReport AssessmentReport) (string, string) { if assessmentReport.MigrationComplexity != "" { - return assessmentReport.MigrationComplexity + return assessmentReport.MigrationComplexity, assessmentReport.MigrationComplexityExplanation } counts := lo.CountValuesBy(assessmentReport.Issues, func(issue AssessmentIssue) string { @@ -97,7 +95,7 @@ func calculateMigrationComplexityForPG(assessmentReport AssessmentReport) string } migrationComplexityRationale = buildRationale(finalComplexity, l1IssueCount, l2IssueCount, l3IssueCount) - return finalComplexity + return finalComplexity, migrationComplexityRationale } // This is a temporary logic to get migration complexity for oracle based on the migration level from ora2pg report. @@ -212,92 +210,12 @@ func getComplexityForLevel(level string, count int) string { } // ======================================= Migration Complexity Explanation ========================================== - -// TODO: discuss if the html should be in main report or here -const explainTemplateHTML = ` -{{- if .Summaries }} -

Below is a breakdown of the issues detected in different categories for each impact level.

- - - - - - - - - - - - {{- range .Summaries }} - - - - - - - - {{- end }} - -
CategoryLevel 1Level 2Level 3Total
{{ .Category }}{{ index .ImpactCounts "LEVEL_1" }}{{ index .ImpactCounts "LEVEL_2" }}{{ index .ImpactCounts "LEVEL_3" }}{{ .TotalIssueCount }}
-{{- end }} - -

- Complexity: {{ .Complexity }}
- Reasoning: {{ .ComplexityRationale }} -

- -

-Impact Levels:
- Level 1: Resolutions are available with minimal effort.
- Level 2: Resolutions are available requiring moderate effort.
- Level 3: Resolutions may not be available or are complex. -

-` - -const explainTemplateText = `{{ .ComplexityRationale }}` - -type MigrationComplexityExplanationData struct { - Summaries []MigrationComplexityCategorySummary - Complexity string - ComplexityRationale string // short reasoning or explanation text -} - type MigrationComplexityCategorySummary struct { Category string TotalIssueCount int ImpactCounts map[string]int // e.g. {"Level-1": 3, "Level-2": 5, "Level-3": 2} } -func buildMigrationComplexityExplanation(sourceDBType string, assessmentReport AssessmentReport, reportFormat string) (string, error) { - if sourceDBType != POSTGRESQL { - return "", nil - } - - var explanation MigrationComplexityExplanationData - explanation.Complexity = assessmentReport.MigrationComplexity - explanation.ComplexityRationale = migrationComplexityRationale - - explanation.Summaries = buildCategorySummary(assessmentReport.Issues) - - var tmpl *template.Template - var err error - if reportFormat == "html" { - tmpl, err = template.New("Explain").Parse(explainTemplateHTML) - } else { - tmpl, err = template.New("Explain").Parse(explainTemplateText) - } - - if err != nil { - return "", fmt.Errorf("failed creating the explanation template: %w", err) - } - - var buf bytes.Buffer - if err := tmpl.Execute(&buf, explanation); err != nil { - return "", fmt.Errorf("failed executing the template with data: %w", err) - } - return buf.String(), nil -} - func buildRationale(finalComplexity string, l1Count int, l2Count int, l3Count int) string { switch finalComplexity { case constants.MIGRATION_COMPLEXITY_HIGH: @@ -310,10 +228,11 @@ func buildRationale(finalComplexity string, l1Count int, l2Count int, l3Count in return "" } -func buildCategorySummary(issues []AssessmentIssue) []MigrationComplexityCategorySummary { +func buildCategorySummary(sourceDBType string, issues []AssessmentIssue) []MigrationComplexityCategorySummary { if len(issues) == 0 { return nil - + } else if sourceDBType != POSTGRESQL { + return nil } summaryMap := make(map[string]*MigrationComplexityCategorySummary) diff --git a/yb-voyager/cmd/templates/migration_assessment_report.template b/yb-voyager/cmd/templates/migration_assessment_report.template index 3e177323ff..1c698a5f37 100644 --- a/yb-voyager/cmd/templates/migration_assessment_report.template +++ b/yb-voyager/cmd/templates/migration_assessment_report.template @@ -274,8 +274,44 @@ {{if and (ne .MigrationComplexity "NOT AVAILABLE") (ne (len .MigrationComplexityExplanation) 0)}}

Migration Complexity Explanation

-

{{ .MigrationComplexityExplanation }}

- {{end}} + {{- if .MigrationComplexityCategorySummary }} +

Below is a breakdown of the issues detected in different categories for each impact level.

+ + + + + + + + + + + + {{ range $idx, $summary := .MigrationComplexityCategorySummary }} + + + + + + + + {{- end }} + +
CategoryLevel 1Level 2Level 3Total
{{ $summary.Category }}{{ index $summary.ImpactCounts "LEVEL_1" }}{{ index $summary.ImpactCounts "LEVEL_2" }}{{ index $summary.ImpactCounts "LEVEL_3" }}{{ $summary.TotalIssueCount }}
+ {{- end }} + +

+ Complexity: {{ .MigrationComplexity }}
+ Reasoning: {{ .MigrationComplexityExplanation }} +

+ +

+ Impact Levels:
+ Level 1: Resolutions are available with minimal effort.
+ Level 2: Resolutions are available requiring moderate effort.
+ Level 3: Resolutions may not be available or are complex. +

+ {{end}}

Assessment Issues

{{ if .Issues }} diff --git a/yb-voyager/src/callhome/diagnostics.go b/yb-voyager/src/callhome/diagnostics.go index 988f0425c4..e728e44873 100644 --- a/yb-voyager/src/callhome/diagnostics.go +++ b/yb-voyager/src/callhome/diagnostics.go @@ -272,7 +272,7 @@ func SendPayload(payload *Payload) error { requestBody := bytes.NewBuffer(postBody) log.Infof("callhome: Payload being sent for diagnostic usage: %s\n", string(postBody)) - callhomeURL := fmt.Sprintf("https://%s:%d/", CALL_HOME_SERVICE_HOST, CALL_HOME_SERVICE_PORT) + callhomeURL := fmt.Sprintf("http://%s:%d/", CALL_HOME_SERVICE_HOST, CALL_HOME_SERVICE_PORT) resp, err := http.Post(callhomeURL, "application/json", requestBody) if err != nil { log.Infof("error while sending diagnostic data: %s", err)