From 6bf1539e0c78361f63ce4e46f075f281940670fe Mon Sep 17 00:00:00 2001 From: Sanyam Singhal Date: Thu, 23 Jan 2025 17:30:36 +0530 Subject: [PATCH] Enhancements to YBD payload after assessment refactoring (#2238) * Bumped up the payload version to 1.2 * Updated the AssessmentIssue payload for YBD to be in sync with standard struct format * Adding MigrationComplexityExplanation to ybd assessment payload --- yb-voyager/cmd/assessMigrationCommand.go | 124 +++++++---------------- yb-voyager/cmd/common.go | 38 ++++--- 2 files changed, 60 insertions(+), 102 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f89a5fba7..074b6b5f1 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -166,7 +166,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) { // whatever happens later will be stored in the struct's field. so to be on safer side, we will build it again here as per required format. explanation, err := buildMigrationComplexityExplanation(source.DBType, assessmentReport, "") if err != nil { - log.Errorf("failed to build migration complexity explanation: %v", err) + log.Errorf("failed to build migration complexity explanation for callhome assessment payload: %v", err) } var obfuscatedIssues []callhome.AssessmentIssueCallhome @@ -462,15 +462,22 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE utils.PrintAndLog("failed to calculate the total sharded table size from tableIndexStats: %v", err) } - assessmentIssues := flattenAssessmentReportToAssessmentIssues(assessmentReport) + assessmentIssues := convertAssessmentIssueToYugabyteDAssessmentIssue(assessmentReport) + // we will build this twice for json and html reports both, at the time of report generation. + // whatever happens later will be stored in the struct's field. so to be on safer side, we will build it again here as per required format. + explanation, err := buildMigrationComplexityExplanation(source.DBType, assessmentReport, "") + if err != nil { + log.Errorf("failed to build migration complexity explanation for yugabyted assessment payload: %v", err) + } payload := AssessMigrationPayload{ - PayloadVersion: ASSESS_MIGRATION_YBD_PAYLOAD_VERSION, - VoyagerVersion: assessmentReport.VoyagerVersion, - TargetDBVersion: assessmentReport.TargetDBVersion, - MigrationComplexity: assessmentReport.MigrationComplexity, - SchemaSummary: assessmentReport.SchemaSummary, - AssessmentIssues: assessmentIssues, + PayloadVersion: ASSESS_MIGRATION_YBD_PAYLOAD_VERSION, + VoyagerVersion: assessmentReport.VoyagerVersion, + TargetDBVersion: assessmentReport.TargetDBVersion, + MigrationComplexity: assessmentReport.MigrationComplexity, + MigrationComplexityExplanation: explanation, + SchemaSummary: assessmentReport.SchemaSummary, + AssessmentIssues: assessmentIssues, SourceSizeDetails: SourceDBSizeDetails{ TotalIndexSize: assessmentReport.GetTotalIndexSize(), TotalTableSize: assessmentReport.GetTotalTableSize(), @@ -510,84 +517,25 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE return ev } -// flatten UnsupportedDataTypes, UnsupportedFeatures, MigrationCaveats -func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []AssessmentIssueYugabyteD { - var issues []AssessmentIssueYugabyteD - - var dataTypesDocsLink string - switch source.DBType { - case POSTGRESQL: - dataTypesDocsLink = UNSUPPORTED_DATATYPES_DOC_LINK - case ORACLE: - dataTypesDocsLink = UNSUPPORTED_DATATYPES_DOC_LINK_ORACLE - } - for _, unsupportedDataType := range ar.UnsupportedDataTypes { - issues = append(issues, AssessmentIssueYugabyteD{ - Type: constants.DATATYPE, - TypeDescription: GetCategoryDescription(constants.DATATYPE), - Subtype: unsupportedDataType.DataType, - ObjectName: fmt.Sprintf("%s.%s.%s", unsupportedDataType.SchemaName, unsupportedDataType.TableName, unsupportedDataType.ColumnName), - SqlStatement: "", - DocsLink: dataTypesDocsLink, - }) - } - - for _, unsupportedFeature := range ar.UnsupportedFeatures { - for _, object := range unsupportedFeature.Objects { - issues = append(issues, AssessmentIssueYugabyteD{ - Type: constants.FEATURE, - TypeDescription: GetCategoryDescription(constants.FEATURE), - Subtype: unsupportedFeature.FeatureName, - SubtypeDescription: unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features - ObjectName: object.ObjectName, - SqlStatement: object.SqlStatement, - DocsLink: unsupportedFeature.DocsLink, - MinimumVersionsFixedIn: unsupportedFeature.MinimumVersionsFixedIn, - }) - } - } - - for _, migrationCaveat := range ar.MigrationCaveats { - for _, object := range migrationCaveat.Objects { - issues = append(issues, AssessmentIssueYugabyteD{ - Type: constants.MIGRATION_CAVEATS, - TypeDescription: GetCategoryDescription(constants.MIGRATION_CAVEATS), - Subtype: migrationCaveat.FeatureName, - SubtypeDescription: migrationCaveat.FeatureDescription, - ObjectName: object.ObjectName, - SqlStatement: object.SqlStatement, - DocsLink: migrationCaveat.DocsLink, - MinimumVersionsFixedIn: migrationCaveat.MinimumVersionsFixedIn, - }) - } - } - - for _, uqc := range ar.UnsupportedQueryConstructs { - issues = append(issues, AssessmentIssueYugabyteD{ - Type: constants.QUERY_CONSTRUCT, - TypeDescription: GetCategoryDescription(constants.QUERY_CONSTRUCT), - Subtype: uqc.ConstructTypeName, - SqlStatement: uqc.Query, - DocsLink: uqc.DocsLink, - MinimumVersionsFixedIn: uqc.MinimumVersionsFixedIn, - }) - } - - for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects { - for _, object := range plpgsqlObjects.Objects { - issues = append(issues, AssessmentIssueYugabyteD{ - Type: constants.PLPGSQL_OBJECT, - TypeDescription: GetCategoryDescription(constants.PLPGSQL_OBJECT), - Subtype: plpgsqlObjects.FeatureName, - SubtypeDescription: plpgsqlObjects.FeatureDescription, - ObjectName: object.ObjectName, - SqlStatement: object.SqlStatement, - DocsLink: plpgsqlObjects.DocsLink, - MinimumVersionsFixedIn: plpgsqlObjects.MinimumVersionsFixedIn, - }) +func convertAssessmentIssueToYugabyteDAssessmentIssue(ar AssessmentReport) []AssessmentIssueYugabyteD { + var result []AssessmentIssueYugabyteD + for _, issue := range ar.Issues { + ybdIssue := AssessmentIssueYugabyteD{ + Category: issue.Category, + CategoryDescription: issue.CategoryDescription, + Type: issue.Type, // Ques: should we be just sending Name in AssessmentIssueYugabyteD payload + Name: issue.Name, + Description: issue.Description, + Impact: issue.Impact, + ObjectType: issue.ObjectType, + ObjectName: issue.ObjectName, + SqlStatement: issue.SqlStatement, + DocsLink: issue.DocsLink, + MinimumVersionsFixedIn: issue.MinimumVersionsFixedIn, } + result = append(result, ybdIssue) } - return issues + return result } func runAssessment() error { @@ -1229,10 +1177,10 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U }) } feature := UnsupportedFeature{ - FeatureName: issueName, - DisplayDDL: true, - DocsLink: docsLink, - Objects: objects, + FeatureName: issueName, + DisplayDDL: true, + DocsLink: docsLink, + Objects: objects, MinimumVersionsFixedIn: minVersionsFixedIn, } unsupportedPlpgSqlObjects = append(unsupportedPlpgSqlObjects, feature) diff --git a/yb-voyager/cmd/common.go b/yb-voyager/cmd/common.go index 36da5774a..364ed7a6d 100644 --- a/yb-voyager/cmd/common.go +++ b/yb-voyager/cmd/common.go @@ -1129,27 +1129,37 @@ type AssessMigrationDBConfig struct { // =============== for yugabyted controlplane ==============// // TODO: see if this can be accommodated in controlplane pkg, facing pkg cyclic dependency issue -var ASSESS_MIGRATION_YBD_PAYLOAD_VERSION = "1.1" // version(s) till now: 1.0, 1.1 +/* +Version History +1.0: Introduced AssessmentIssue field for storing assessment issues in flattened format +1.1: Added TargetDBVersion and AssessmentIssueYugabyteD.MinimumVersionFixedIn +1.2: Syncing it with original AssessmentIssue(adding fields Category, CategoryDescription, Type, Name, Description, Impact, ObjectType) and MigrationComplexityExplanation; +*/ +var ASSESS_MIGRATION_YBD_PAYLOAD_VERSION = "1.2" type AssessMigrationPayload struct { - PayloadVersion string - VoyagerVersion string - TargetDBVersion *ybversion.YBVersion - MigrationComplexity string - SchemaSummary utils.SchemaSummary - AssessmentIssues []AssessmentIssueYugabyteD - SourceSizeDetails SourceDBSizeDetails - TargetRecommendations TargetSizingRecommendations - ConversionIssues []utils.AnalyzeSchemaIssue + PayloadVersion string + VoyagerVersion string + TargetDBVersion *ybversion.YBVersion + MigrationComplexity string + MigrationComplexityExplanation string + SchemaSummary utils.SchemaSummary + AssessmentIssues []AssessmentIssueYugabyteD + SourceSizeDetails SourceDBSizeDetails + TargetRecommendations TargetSizingRecommendations + ConversionIssues []utils.AnalyzeSchemaIssue // Depreacted: AssessmentJsonReport is depricated; use the fields directly inside struct AssessmentJsonReport AssessmentReportYugabyteD } type AssessmentIssueYugabyteD struct { - Type string `json:"Type"` // Feature, DataType, MigrationCaveat, UQC - TypeDescription string `json:"TypeDescription"` // Based on AssessmentIssue type - Subtype string `json:"Subtype"` // GIN Indexes, Advisory Locks etc - SubtypeDescription string `json:"SubtypeDescription"` // description based on subtype + Category string `json:"Category"` // expected values: unsupported_features, unsupported_query_constructs, migration_caveats, unsupported_plpgsql_objects, unsupported_datatype + CategoryDescription string `json:"CategoryDescription"` + Type string `json:"Type"` // Ex: GIN_INDEXES, SECURITY_INVOKER_VIEWS, STORED_GENERATED_COLUMNS + Name string `json:"Name"` // Ex: GIN Indexes, Security Invoker Views, Stored Generated Columns + Description string `json:"Description"` // description based on type/name + Impact string `json:"Impact"` // // Level-1, Level-2, Level-3 (no default: need to be assigned for each issue) + ObjectType string `json:"ObjectType"` // For datatype category, ObjectType will be datatype (for eg "geometry") ObjectName string `json:"ObjectName"` // Fully qualified object name(empty if NA, eg UQC) SqlStatement string `json:"SqlStatement"` // DDL or DML(UQC) DocsLink string `json:"DocsLink"` // docs link based on the subtype