From 31bad8197981f79fb5293e600a7e099a7c842999 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 17 Jul 2022 17:23:31 +0200 Subject: [PATCH] Use external lib to convert yaml to json (#1028) this move shared/yml/* into an independent lib --- go.mod | 7 +- go.sum | 12 ++- pipeline/frontend/yaml/compiler/params.go | 7 +- pipeline/schema/schema.go | 9 +- shared/yml/yml.go | 103 ---------------------- shared/yml/yml_test.go | 45 ---------- 6 files changed, 20 insertions(+), 163 deletions(-) delete mode 100644 shared/yml/yml.go delete mode 100644 shared/yml/yml_test.go diff --git a/go.mod b/go.mod index eeb714cd1..c81c3aaf5 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71 + codeberg.org/6543/go-yaml2json v0.1.0 github.com/bmatcuk/doublestar/v4 v4.0.2 github.com/docker/cli v20.10.14+incompatible github.com/docker/distribution v2.8.1+incompatible @@ -29,7 +30,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.1 github.com/rs/zerolog v1.26.1 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.8.0 github.com/tevino/abool v1.2.0 github.com/urfave/cli/v2 v2.5.1 github.com/xanzy/go-gitlab v0.64.0 @@ -40,7 +41,7 @@ require ( golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 xorm.io/builder v0.3.10 xorm.io/xorm v1.3.0 ) @@ -93,7 +94,7 @@ require ( github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/objx v0.4.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ugorji/go/codec v1.2.7 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect diff --git a/go.sum b/go.sum index a03573fa8..0b7890f2a 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71 h1:+ZqwhfAftVOAd7AcLpfh4LBdTeJIyt60vGU39zhQPyA= code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71/go.mod h1:MuMGvUxT8BmFHa0gHhHsrnz91QfmziXuFffm9AuhMCo= +codeberg.org/6543/go-yaml2json v0.1.0 h1:njuf3a8QgsmBXJFiH+7wNR01biBS4MU+XeWE7W3bnus= +codeberg.org/6543/go-yaml2json v0.1.0/go.mod h1:mz61q14LWF4ZABrgMEDMmk3t9dPi6zgR1uBh2VKV2RQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= @@ -602,16 +604,17 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= -github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA= @@ -1060,8 +1063,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pipeline/frontend/yaml/compiler/params.go b/pipeline/frontend/yaml/compiler/params.go index 88f8906be..49f4ac2e5 100644 --- a/pipeline/frontend/yaml/compiler/params.go +++ b/pipeline/frontend/yaml/compiler/params.go @@ -6,9 +6,8 @@ import ( "strconv" "strings" + "codeberg.org/6543/go-yaml2json" "gopkg.in/yaml.v3" - - "github.com/woodpecker-ci/woodpecker/shared/yml" ) // paramsToEnv uses reflection to convert a map[string]interface to a list @@ -77,7 +76,7 @@ func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error } } ymlOut, _ := yaml.Marshal(vv.Interface()) - out, _ := yml.ToJSON(ymlOut) + out, _ := yaml2json.Covnert(ymlOut) return string(out), nil case reflect.Slice, reflect.Array: @@ -103,7 +102,7 @@ func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error if err != nil { return "", err } - out, err = yml.ToJSON(out) + out, err = yaml2json.Covnert(out) if err != nil { return "", err } diff --git a/pipeline/schema/schema.go b/pipeline/schema/schema.go index 34c229e09..094b86629 100644 --- a/pipeline/schema/schema.go +++ b/pipeline/schema/schema.go @@ -1,13 +1,13 @@ package schema import ( + "bytes" _ "embed" "fmt" "io" + "codeberg.org/6543/go-yaml2json" "github.com/xeipuuv/gojsonschema" - - "github.com/woodpecker-ci/woodpecker/shared/yml" ) //go:embed schema.json @@ -16,12 +16,13 @@ var schemaDefinition []byte // Lint lints an io.Reader against the Woodpecker schema.json func Lint(r io.Reader) ([]gojsonschema.ResultError, error) { schemaLoader := gojsonschema.NewBytesLoader(schemaDefinition) - j, err := yml.LoadYmlReaderAsJSON(r) + buff := new(bytes.Buffer) + err := yaml2json.StreamConverter(r, buff) if err != nil { return nil, fmt.Errorf("Failed to load yml file %w", err) } - documentLoader := gojsonschema.NewBytesLoader(j) + documentLoader := gojsonschema.NewBytesLoader(buff.Bytes()) result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { return nil, fmt.Errorf("Validation failed %w", err) diff --git a/shared/yml/yml.go b/shared/yml/yml.go deleted file mode 100644 index dea05e260..000000000 --- a/shared/yml/yml.go +++ /dev/null @@ -1,103 +0,0 @@ -package yml - -import ( - "encoding/json" - "fmt" - "io" - "strconv" - - "gopkg.in/yaml.v3" -) - -// toJSON convert gopkg.in/yaml.v3 nodes to object that can be serialized as json -// fmt.Sprint() with default formatting is used to convert the key to a string key. -func toJSON(node *yaml.Node) (interface{}, error) { - switch node.Kind { - case yaml.DocumentNode: - return toJSON(node.Content[0]) - - case yaml.SequenceNode: - val := make([]interface{}, len(node.Content)) - var err error - for i := range node.Content { - if val[i], err = toJSON(node.Content[i]); err != nil { - return nil, err - } - } - return val, nil - - case yaml.MappingNode: - if (len(node.Content) % 2) != 0 { - return nil, fmt.Errorf("broken mapping node") - } - val := make(map[string]interface{}, len(node.Content)%2) - for i := len(node.Content); i > 1; i = i - 2 { - k, err := toJSON(node.Content[i-2]) - if err != nil { - return nil, err - } - if val[fmt.Sprint(k)], err = toJSON(node.Content[i-1]); err != nil { - return nil, err - } - } - return val, nil - - case yaml.ScalarNode: - switch node.Tag { - case nullTag: - return nil, nil - case boolTag: - return strconv.ParseBool(node.Value) - case intTag: - return strconv.ParseInt(node.Value, 10, 64) - case floatTag: - return strconv.ParseFloat(node.Value, 64) - } - return node.Value, nil - } - - return nil, fmt.Errorf("do not support yaml node kind '%v'", node.Kind) -} - -// ToJSON converts YAML bytes to JSON -func ToJSON(data []byte) ([]byte, error) { - m := &yaml.Node{} - if err := yaml.Unmarshal(data, m); err != nil { - return nil, err - } - - d, err := toJSON(m) - if err != nil { - return nil, err - } - return json.Marshal(d) -} - -// LoadYmlReaderAsJSON reads from an io.Reader containing YAML and converts it to JSON -func LoadYmlReaderAsJSON(r io.Reader) (j []byte, err error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, err - } - - j, err = ToJSON(data) - if err != nil { - return nil, err - } - - return j, nil -} - -// Source: https://github.com/go-yaml/yaml/blob/3e3283e801afc229479d5fc68aa41df1137b8394/resolve.go#L70-L81 -const ( - nullTag = "!!null" - boolTag = "!!bool" - intTag = "!!int" - floatTag = "!!float" - // strTag = "!!str" // we dont have to parse it - // timestampTag = "!!timestamp" // TODO: do we have to parse this? - // seqTag = "!!seq" // TODO: do we have to parse this? - // mapTag = "!!map" // TODO: do we have to parse this? - // binaryTag = "!!binary" // TODO: do we have to parse this? - // mergeTag = "!!merge" // TODO: do we have to parse this? -) diff --git a/shared/yml/yml_test.go b/shared/yml/yml_test.go deleted file mode 100644 index 150b534da..000000000 --- a/shared/yml/yml_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package yml - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestToJSON(t *testing.T) { - tests := []struct { - yaml string - json string - }{{ - yaml: `- name: Jack -- name: Jill -`, - json: `[{"name":"Jack"},{"name":"Jill"}]`, - }, { - yaml: `name: Jack`, - json: `{"name":"Jack"}`, - }, { - yaml: `name: Jack -job: Butcher -`, - json: `{"job":"Butcher","name":"Jack"}`, - }, { - yaml: `- name: Jack - job: Butcher -- name: Jill - job: Cook - obj: - empty: false - data: | - some data 123 - with new line -`, - json: `[{"job":"Butcher","name":"Jack"},{"job":"Cook","name":"Jill","obj":{"data":"some data 123\nwith new line\n","empty":false}}]`, - }} - - for _, tc := range tests { - result, err := ToJSON([]byte(tc.yaml)) - assert.NoError(t, err) - assert.EqualValues(t, tc.json, string(result)) - } -}