Skip to content

Commit

Permalink
Refactor migration assessment codepath: multiple minor changes (#2137)
Browse files Browse the repository at this point in the history 
* Renamed AssessmentIssuePayload struct used for ybd to AssessmentIssueYugabyteD
* Rename struct: Issue to AnalyzeSchemaIssue

* Introduced AssessmentIssue struct for flattening the issues reported in AssessmentReport struct
* fixing the struct in the unit test for checking non breaking upgrade
  • Loading branch information
@sanyamsinghal
sanyamsinghal authored Jan 6, 2025
1 parent 26fc27b commit 953591c
Show file tree
Hide file tree
Showing 11 changed files with 163 additions and 157 deletions.
60 changes: 22 additions & 38 deletions yb-voyager/cmd/analyzeSchema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,43 +194,34 @@ var (
)

const (
CONVERSION_ISSUE_REASON = "CREATE CONVERSION is not supported yet"
GIN_INDEX_MULTI_COLUMN_ISSUE_REASON = "Schema contains gin index on multi column which is not supported."
ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON = "Adding primary key to a partitioned table is not supported yet."
INHERITANCE_ISSUE_REASON = "TABLE INHERITANCE not supported in YugabyteDB"
CONSTRAINT_TRIGGER_ISSUE_REASON = "CONSTRAINT TRIGGER not supported yet."
REFERENCING_CLAUSE_FOR_TRIGGERS = "REFERENCING clause (transition tables) not supported yet."
BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE = "Partitioned tables cannot have BEFORE / FOR EACH ROW triggers."
COMPOUND_TRIGGER_ISSUE_REASON = "COMPOUND TRIGGER not supported in YugabyteDB."

STORED_GENERATED_COLUMN_ISSUE_REASON = "Stored generated columns are not supported."
UNSUPPORTED_EXTENSION_ISSUE = "This extension is not supported in YugabyteDB by default."
EXCLUSION_CONSTRAINT_ISSUE = "Exclusion constraint is not supported yet"
ALTER_TABLE_DISABLE_RULE_ISSUE = "ALTER TABLE name DISABLE RULE not supported yet"
STORAGE_PARAMETERS_DDL_STMT_ISSUE = "Storage parameters are not supported yet."
ALTER_TABLE_SET_ATTRIBUTE_ISSUE = "ALTER TABLE .. ALTER COLUMN .. SET ( attribute = value ) not supported yet"
FOREIGN_TABLE_ISSUE_REASON = "Foreign tables require manual intervention."
ALTER_TABLE_CLUSTER_ON_ISSUE = "ALTER TABLE CLUSTER not supported yet."
DEFERRABLE_CONSTRAINT_ISSUE = "DEFERRABLE constraints not supported yet"
POLICY_ROLE_ISSUE = "Policy require roles to be created."
VIEW_CHECK_OPTION_ISSUE = "Schema containing VIEW WITH CHECK OPTION is not supported yet."
ISSUE_INDEX_WITH_COMPLEX_DATATYPES = `INDEX on column '%s' not yet supported`
ISSUE_PK_UK_CONSTRAINT_WITH_COMPLEX_DATATYPES = `Primary key and Unique constraint on column '%s' not yet supported`
ISSUE_UNLOGGED_TABLE = "UNLOGGED tables are not supported yet."
// Issues detected using regexp, reported in assessment and analyze both
CONVERSION_ISSUE_REASON = "CREATE CONVERSION is not supported yet"
UNSUPPORTED_EXTENSION_ISSUE = "This extension is not supported in YugabyteDB by default."
VIEW_CHECK_OPTION_ISSUE = "Schema containing VIEW WITH CHECK OPTION is not supported yet."

// Refactor: constants below used in some comparisions (use Issue Type there and remove these)
INHERITANCE_ISSUE_REASON = "TABLE INHERITANCE not supported in YugabyteDB"
ADDING_PK_TO_PARTITIONED_TABLE_ISSUE_REASON = "Adding primary key to a partitioned table is not supported yet."
COMPOUND_TRIGGER_ISSUE_REASON = "COMPOUND TRIGGER not supported in YugabyteDB."
STORED_GENERATED_COLUMN_ISSUE_REASON = "Stored generated columns are not supported."
FOREIGN_TABLE_ISSUE_REASON = "Foreign tables require manual intervention."
DEFERRABLE_CONSTRAINT_ISSUE = "DEFERRABLE constraints not supported yet"
POLICY_ROLE_ISSUE = "Policy require roles to be created."
ISSUE_INDEX_WITH_COMPLEX_DATATYPES = `INDEX on column '%s' not yet supported`
INDEX_METHOD_ISSUE_REASON = "Schema contains %s index which is not supported."

UNSUPPORTED_DATATYPE = "Unsupported datatype"
UNSUPPORTED_DATATYPE_LIVE_MIGRATION = "Unsupported datatype for Live migration"
UNSUPPORTED_DATATYPE_LIVE_MIGRATION_WITH_FF_FB = "Unsupported datatype for Live migration with fall-forward/fallback"
UNSUPPORTED_PG_SYNTAX = "Unsupported PG syntax"

INDEX_METHOD_ISSUE_REASON = "Schema contains %s index which is not supported."
INSUFFICIENT_COLUMNS_IN_PK_FOR_PARTITION = "insufficient columns in the PRIMARY KEY constraint definition in CREATE TABLE"
GIN_INDEX_DETAILS = "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported."
SECURITY_INVOKER_VIEWS_ISSUE = "Security Invoker Views not supported yet"
)

// Reports one case in JSON
func reportCase(filePath string, reason string, ghIssue string, suggestion string, objType string, objName string, sqlStmt string, issueType string, docsLink string) {
var issue utils.Issue
var issue utils.AnalyzeSchemaIssue
issue.FilePath = filePath
issue.Reason = reason
issue.GH = ghIssue
Expand Down Expand Up @@ -354,13 +345,6 @@ func checkStmtsUsingParser(sqlInfoArr []sqlInfo, fpath string, objType string) {
// Checks compatibility of views
func checkViews(sqlInfoArr []sqlInfo, fpath string) {
for _, sqlInfo := range sqlInfoArr {
/*if dropMatViewRegex.MatchString(sqlInfo.stmt) {
reportCase(fpath, "DROP MATERIALIZED VIEW not supported yet.",a
"https://github.com/YugaByte/yugabyte-db/issues/10102", "")
} else if view := matViewRegex.FindStringSubmatch(sqlInfo.stmt); view != nil {
reportCase(fpath, "Schema contains materialized view which is not supported. The view is: "+view[1],
"https://github.com/yugabyte/yugabyte-db/issues/10102", "")
} else */
if view := viewWithCheckRegex.FindStringSubmatch(sqlInfo.stmt); view != nil {
summaryMap["VIEW"].invalidCount[sqlInfo.objName] = true
reportCase(fpath, VIEW_CHECK_OPTION_ISSUE, "https://github.com/yugabyte/yugabyte-db/issues/22716",
Expand Down Expand Up @@ -635,7 +619,7 @@ var MigrationCaveatsIssues = []string{
UNSUPPORTED_DATATYPE_LIVE_MIGRATION_WITH_FF_FB,
}

func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fileName string, isPlPgSQLIssue bool) utils.Issue {
func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fileName string, isPlPgSQLIssue bool) utils.AnalyzeSchemaIssue {
issueType := UNSUPPORTED_FEATURES
switch true {
case isPlPgSQLIssue:
Expand Down Expand Up @@ -685,15 +669,15 @@ func convertIssueInstanceToAnalyzeIssue(issueInstance queryissue.QueryIssue, fil

summaryMap[issueInstance.ObjectType].invalidCount[issueInstance.ObjectName] = true

return utils.Issue{
return utils.AnalyzeSchemaIssue{
IssueType: issueType,
ObjectType: issueInstance.ObjectType,
ObjectName: displayObjectName,
Reason: issueInstance.TypeName,
Type: issueInstance.Type,
SqlStatement: issueInstance.SqlStatement,
DocsLink: issueInstance.DocsLink,
FilePath: fileName,
IssueType: issueType,
Suggestion: issueInstance.Suggestion,
GH: issueInstance.GH,
MinimumVersionsFixedIn: issueInstance.MinimumVersionsFixedIn,
Expand Down Expand Up @@ -1090,7 +1074,7 @@ func analyzeSchemaInternal(sourceDBConf *srcdb.Source, detectIssues bool) utils.

// Ideally all filtering of issues should happen in queryissue pkg layer,
// but until we move all issue detection logic to queryissue pkg, we will filter issues here as well.
schemaAnalysisReport.Issues = lo.Filter(schemaAnalysisReport.Issues, func(i utils.Issue, index int) bool {
schemaAnalysisReport.Issues = lo.Filter(schemaAnalysisReport.Issues, func(i utils.AnalyzeSchemaIssue, index int) bool {
fixed, err := i.IsFixedIn(targetDbVersion)
if err != nil {
utils.ErrExit("checking if issue %v is supported: %v", i, err)
Expand Down Expand Up @@ -1254,7 +1238,7 @@ func packAndSendAnalyzeSchemaPayload(status string) {
payload := createCallhomePayload()

payload.MigrationPhase = ANALYZE_PHASE
var callhomeIssues []utils.Issue
var callhomeIssues []utils.AnalyzeSchemaIssue
for _, issue := range schemaAnalysisReport.Issues {
issue.SqlStatement = "" // Obfuscate sensitive information before sending to callhome cluster
if !lo.ContainsBy(reasonsToSendObjectNameToCallhome, func(r string) bool {
Expand Down
Loading

0 comments on commit 953591c

Please sign in to comment.