mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-06-05 15:09:26 +00:00
Code Issues Projects Releases Wiki Activity

fix sqlite migration on column drop of uncommon schemas (#629)

Browse source 
* fix sqlite schema normalising

* sqlite robuster alg to extract columns form schema
This commit is contained in:
6543 2021-12-20 09:59:27 +01:00 committed by GitHub
parent ce5247c675
commit b2ce1f5da5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 91 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
server/store/datastore/migration/common.go
View file

@ -82,15 +82,13 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
if err != nil {
return err
}
tableSQL := string(res[0]["sql"])
tableSQL := normalizeSQLiteTableSchema(string(res[0]["sql"]))
// Separate out the column definitions
tableSQL = tableSQL[strings.Index(tableSQL, "("):]
// Remove the required columnNames
for _, name := range columnNames {
tableSQL = regexp.MustCompile(regexp.QuoteMeta("`"+name+"`")+"[^`,)]*?[,)]").ReplaceAllString(tableSQL, "")
}
tableSQL = removeColumnFromSQLITETableSchema(tableSQL, columnNames...)
// Ensure the query is ended properly
tableSQL = strings.TrimSpace(tableSQL)
@ -102,7 +100,16 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
}
// Find all the columns in the table
columns := regexp.MustCompile("`([^`]*)`").FindAllString(tableSQL, -1)
var columns []string
for _, rawColumn := range strings.Split(strings.ReplaceAll(tableSQL[1:len(tableSQL)-1], ", ", ",\n"), "\n") {
if strings.ContainsAny(rawColumn, "()") {
continue
}
rawColumn = strings.TrimSpace(rawColumn)
columns = append(columns,
strings.ReplaceAll(rawColumn[0:strings.Index(rawColumn, " ")], "`", ""),
)
}
tableSQL = fmt.Sprintf("CREATE TABLE `new_%s_new` ", tableName) + tableSQL
if _, err := sess.Exec(tableSQL); err != nil {
@ -204,3 +211,31 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
return nil
}
var whitespaces = regexp.MustCompile(`\s+`)
var columnSeparator = regexp.MustCompile(`\s?,\s?`)
func removeColumnFromSQLITETableSchema(schema string, names ...string) string {
if len(names) == 0 {
return schema
}
for i := range names {
if len(names[i]) == 0 {
continue
}
schema = regexp.MustCompile(`\s(`+
regexp.QuoteMeta("`"+names[i]+"`")+
"|"+
regexp.QuoteMeta(names[i])+
")[^`,)]*?[,)]").ReplaceAllString(schema, "")
}
return schema
}
func normalizeSQLiteTableSchema(schema string) string {
return columnSeparator.ReplaceAllString(
whitespaces.ReplaceAllString(
strings.ReplaceAll(schema, "\n", " "),
" "),
", ")
}

51
server/store/datastore/migration/common_test.go Normal file
View file

@ -0,0 +1,51 @@
package migration
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestRemoveColumnFromSQLITETableSchema(t *testing.T) {
schema := "CREATE TABLE repos ( repo_id INTEGER PRIMARY KEY AUTOINCREMENT, repo_user_id INTEGER, repo_owner TEXT, " +
"repo_name TEXT, repo_full_name TEXT, `repo_avatar` TEXT, repo_branch TEXT, repo_timeout INTEGER, " +
"repo_allow_pr BOOLEAN, repo_config_path TEXT, repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, " +
"repo_fallback BOOLEAN, UNIQUE(repo_full_name) )"
assert.EqualValues(t, schema, removeColumnFromSQLITETableSchema(schema, ""))
assert.EqualValues(t, "CREATE TABLE repos ( repo_id INTEGER PRIMARY KEY AUTOINCREMENT, repo_user_id INTEGER, repo_owner TEXT, "+
"repo_name TEXT, repo_full_name TEXT, repo_branch TEXT, repo_timeout INTEGER, "+
"repo_allow_pr BOOLEAN, repo_config_path TEXT, repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, "+
"repo_fallback BOOLEAN, UNIQUE(repo_full_name) )", removeColumnFromSQLITETableSchema(schema, "repo_avatar"))
assert.EqualValues(t, "CREATE TABLE repos ( repo_user_id INTEGER, repo_owner TEXT, "+
"repo_name TEXT, repo_full_name TEXT, `repo_avatar` TEXT, repo_timeout INTEGER, "+
"repo_allow_pr BOOLEAN, repo_config_path TEXT, repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, "+
"repo_fallback BOOLEAN, UNIQUE(repo_full_name) )", removeColumnFromSQLITETableSchema(schema, "repo_id", "repo_branch", "invalid", ""))
}
func TestNormalizeSQLiteTableSchema(t *testing.T) {
assert.EqualValues(t, "", normalizeSQLiteTableSchema(``))
assert.EqualValues(t,
"CREATE TABLE repos ( repo_id INTEGER PRIMARY KEY AUTOINCREMENT, "+
"repo_user_id INTEGER, repo_owner TEXT, repo_name TEXT, repo_full_name TEXT, "+
"`repo_avatar` TEXT, repo_link TEXT, repo_clone TEXT, repo_branch TEXT, "+
"repo_timeout INTEGER, repo_allow_pr BOOLEAN, repo_config_path TEXT, "+
"repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, "+
"repo_fallback BOOLEAN, UNIQUE(repo_full_name) )",
normalizeSQLiteTableSchema(`CREATE TABLE repos (
repo_id INTEGER PRIMARY KEY AUTOINCREMENT
,repo_user_id INTEGER
,repo_owner TEXT,
repo_name TEXT
,repo_full_name TEXT
,`+"`"+`repo_avatar`+"`"+` TEXT
,repo_link TEXT
,repo_clone TEXT
,repo_branch TEXT ,repo_timeout INTEGER
,repo_allow_pr BOOLEAN
,repo_config_path TEXT
, repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, repo_fallback BOOLEAN,UNIQUE(repo_full_name)
)`))
}