mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2024-05-15 04:32:39 +00:00
Use external lib to convert yaml to json (#1028)
this move shared/yml/* into an independent lib
This commit is contained in:
6543
GitHub
parent
86cbd63dc2
commit
31bad81979
7
go.mod
7
go.mod
|
|
@ -4,6 +4,7 @@ go 1.18
|
|||
|
||||
require (
|
||||
code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71
|
||||
codeberg.org/6543/go-yaml2json v0.1.0
|
||||
github.com/bmatcuk/doublestar/v4 v4.0.2
|
||||
github.com/docker/cli v20.10.14+incompatible
|
||||
github.com/docker/distribution v2.8.1+incompatible
|
||||
|
|
@ -29,7 +30,7 @@ require (
|
|||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.12.1
|
||||
github.com/rs/zerolog v1.26.1
|
||||
github.com/stretchr/testify v1.7.1
|
||||
github.com/stretchr/testify v1.8.0
|
||||
github.com/tevino/abool v1.2.0
|
||||
github.com/urfave/cli/v2 v2.5.1
|
||||
github.com/xanzy/go-gitlab v0.64.0
|
||||
|
|
@ -40,7 +41,7 @@ require (
|
|||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
|
||||
google.golang.org/grpc v1.47.0
|
||||
google.golang.org/protobuf v1.28.0
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
xorm.io/builder v0.3.10
|
||||
xorm.io/xorm v1.3.0
|
||||
)
|
||||
|
|
@ -93,7 +94,7 @@ require (
|
|||
github.com/rogpeppe/go-internal v1.8.1 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/stretchr/objx v0.3.0 // indirect
|
||||
github.com/stretchr/objx v0.4.0 // indirect
|
||||
github.com/syndtr/goleveldb v1.0.0 // indirect
|
||||
github.com/ugorji/go/codec v1.2.7 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
|
|
|
|||
12
go.sum
12
go.sum
|
|
@ -32,6 +32,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
|
|||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71 h1:+ZqwhfAftVOAd7AcLpfh4LBdTeJIyt60vGU39zhQPyA=
|
||||
code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71/go.mod h1:MuMGvUxT8BmFHa0gHhHsrnz91QfmziXuFffm9AuhMCo=
|
||||
codeberg.org/6543/go-yaml2json v0.1.0 h1:njuf3a8QgsmBXJFiH+7wNR01biBS4MU+XeWE7W3bnus=
|
||||
codeberg.org/6543/go-yaml2json v0.1.0/go.mod h1:mz61q14LWF4ZABrgMEDMmk3t9dPi6zgR1uBh2VKV2RQ=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
|
||||
gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=
|
||||
|
|
@ -602,16 +604,17 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J
|
|||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
|
||||
github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA=
|
||||
|
|
@ -1060,8 +1063,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
|
||||
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
"codeberg.org/6543/go-yaml2json"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/woodpecker-ci/woodpecker/shared/yml"
|
||||
)
|
||||
|
||||
// paramsToEnv uses reflection to convert a map[string]interface to a list
|
||||
|
|
@ -77,7 +76,7 @@ func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error
|
|||
}
|
||||
}
|
||||
ymlOut, _ := yaml.Marshal(vv.Interface())
|
||||
out, _ := yml.ToJSON(ymlOut)
|
||||
out, _ := yaml2json.Covnert(ymlOut)
|
||||
return string(out), nil
|
||||
|
||||
case reflect.Slice, reflect.Array:
|
||||
|
|
@ -103,7 +102,7 @@ func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error
|
|||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err = yml.ToJSON(out)
|
||||
out, err = yaml2json.Covnert(out)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
package schema
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"codeberg.org/6543/go-yaml2json"
|
||||
"github.com/xeipuuv/gojsonschema"
|
||||
|
||||
"github.com/woodpecker-ci/woodpecker/shared/yml"
|
||||
)
|
||||
|
||||
//go:embed schema.json
|
||||
|
|
@ -16,12 +16,13 @@ var schemaDefinition []byte
|
|||
// Lint lints an io.Reader against the Woodpecker schema.json
|
||||
func Lint(r io.Reader) ([]gojsonschema.ResultError, error) {
|
||||
schemaLoader := gojsonschema.NewBytesLoader(schemaDefinition)
|
||||
j, err := yml.LoadYmlReaderAsJSON(r)
|
||||
buff := new(bytes.Buffer)
|
||||
err := yaml2json.StreamConverter(r, buff)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to load yml file %w", err)
|
||||
}
|
||||
|
||||
documentLoader := gojsonschema.NewBytesLoader(j)
|
||||
documentLoader := gojsonschema.NewBytesLoader(buff.Bytes())
|
||||
result, err := gojsonschema.Validate(schemaLoader, documentLoader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Validation failed %w", err)
|
||||
|
|
|
|||
|
|
@ -1,103 +0,0 @@
|
|||
package yml
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// toJSON convert gopkg.in/yaml.v3 nodes to object that can be serialized as json
|
||||
// fmt.Sprint() with default formatting is used to convert the key to a string key.
|
||||
func toJSON(node *yaml.Node) (interface{}, error) {
|
||||
switch node.Kind {
|
||||
case yaml.DocumentNode:
|
||||
return toJSON(node.Content[0])
|
||||
|
||||
case yaml.SequenceNode:
|
||||
val := make([]interface{}, len(node.Content))
|
||||
var err error
|
||||
for i := range node.Content {
|
||||
if val[i], err = toJSON(node.Content[i]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return val, nil
|
||||
|
||||
case yaml.MappingNode:
|
||||
if (len(node.Content) % 2) != 0 {
|
||||
return nil, fmt.Errorf("broken mapping node")
|
||||
}
|
||||
val := make(map[string]interface{}, len(node.Content)%2)
|
||||
for i := len(node.Content); i > 1; i = i - 2 {
|
||||
k, err := toJSON(node.Content[i-2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if val[fmt.Sprint(k)], err = toJSON(node.Content[i-1]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return val, nil
|
||||
|
||||
case yaml.ScalarNode:
|
||||
switch node.Tag {
|
||||
case nullTag:
|
||||
return nil, nil
|
||||
case boolTag:
|
||||
return strconv.ParseBool(node.Value)
|
||||
case intTag:
|
||||
return strconv.ParseInt(node.Value, 10, 64)
|
||||
case floatTag:
|
||||
return strconv.ParseFloat(node.Value, 64)
|
||||
}
|
||||
return node.Value, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("do not support yaml node kind '%v'", node.Kind)
|
||||
}
|
||||
|
||||
// ToJSON converts YAML bytes to JSON
|
||||
func ToJSON(data []byte) ([]byte, error) {
|
||||
m := &yaml.Node{}
|
||||
if err := yaml.Unmarshal(data, m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
d, err := toJSON(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(d)
|
||||
}
|
||||
|
||||
// LoadYmlReaderAsJSON reads from an io.Reader containing YAML and converts it to JSON
|
||||
func LoadYmlReaderAsJSON(r io.Reader) (j []byte, err error) {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
j, err = ToJSON(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return j, nil
|
||||
}
|
||||
|
||||
// Source: https://github.com/go-yaml/yaml/blob/3e3283e801afc229479d5fc68aa41df1137b8394/resolve.go#L70-L81
|
||||
const (
|
||||
nullTag = "!!null"
|
||||
boolTag = "!!bool"
|
||||
intTag = "!!int"
|
||||
floatTag = "!!float"
|
||||
// strTag = "!!str" // we dont have to parse it
|
||||
// timestampTag = "!!timestamp" // TODO: do we have to parse this?
|
||||
// seqTag = "!!seq" // TODO: do we have to parse this?
|
||||
// mapTag = "!!map" // TODO: do we have to parse this?
|
||||
// binaryTag = "!!binary" // TODO: do we have to parse this?
|
||||
// mergeTag = "!!merge" // TODO: do we have to parse this?
|
||||
)
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
package yml
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestToJSON(t *testing.T) {
|
||||
tests := []struct {
|
||||
yaml string
|
||||
json string
|
||||
}{{
|
||||
yaml: `- name: Jack
|
||||
- name: Jill
|
||||
`,
|
||||
json: `[{"name":"Jack"},{"name":"Jill"}]`,
|
||||
}, {
|
||||
yaml: `name: Jack`,
|
||||
json: `{"name":"Jack"}`,
|
||||
}, {
|
||||
yaml: `name: Jack
|
||||
job: Butcher
|
||||
`,
|
||||
json: `{"job":"Butcher","name":"Jack"}`,
|
||||
}, {
|
||||
yaml: `- name: Jack
|
||||
job: Butcher
|
||||
- name: Jill
|
||||
job: Cook
|
||||
obj:
|
||||
empty: false
|
||||
data: |
|
||||
some data 123
|
||||
with new line
|
||||
`,
|
||||
json: `[{"job":"Butcher","name":"Jack"},{"job":"Cook","name":"Jill","obj":{"data":"some data 123\nwith new line\n","empty":false}}]`,
|
||||
}}
|
||||
|
||||
for _, tc := range tests {
|
||||
result, err := ToJSON([]byte(tc.yaml))
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, tc.json, string(result))
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue