diff --git a/go.mod b/go.mod index 493b241d6..3c9c2d20d 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( code.gitea.io/sdk/gitea v0.15.1-0.20221016183512-2d9ee57af1e0 codeberg.org/6543/go-yaml2json v0.3.0 + github.com/alessio/shellescape v1.4.1 github.com/antonmedv/expr v1.9.0 github.com/bmatcuk/doublestar/v4 v4.2.0 github.com/caddyserver/certmagic v0.17.2 @@ -38,7 +39,8 @@ require ( github.com/urfave/cli/v2 v2.20.2 github.com/xanzy/go-gitlab v0.73.1 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/net v0.0.0-20221019024206-cb67ada4b0ad + golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 + golang.org/x/net v0.1.0 golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 golang.org/x/sync v0.1.0 google.golang.org/grpc v1.50.1 @@ -125,13 +127,13 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.23.0 // indirect - golang.org/x/crypto v0.0.0-20221012134737-56aed061732a // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/crypto v0.1.0 // indirect + golang.org/x/mod v0.6.0 // indirect golang.org/x/sys v0.1.0 // indirect - golang.org/x/term v0.0.0-20221017184919-83659145692c // indirect + golang.org/x/term v0.1.0 // indirect golang.org/x/text v0.4.0 // indirect golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/tools v0.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index cd2a5a9e4..1b616f72f 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antonmedv/expr v1.9.0 h1:j4HI3NHEdgDnN9p6oI6Ndr0G5QryMY0FNxT4ONrFDGU= github.com/antonmedv/expr v1.9.0/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= @@ -766,8 +768,8 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20221012134737-56aed061732a h1:NmSIgad6KjE6VvHciPZuNRTKxGhlPfD6OA87W/PLkqg= -golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -778,6 +780,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -799,8 +803,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -846,8 +850,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20221019024206-cb67ada4b0ad h1:Zx6wVVDwwNJFWXNIvDi7o952w3/1ckSwYk/7eykRmjM= -golang.org/x/net v0.0.0-20221019024206-cb67ada4b0ad/go.mod h1:RpDiru2p0u2F0lLpEoqnP2+7xs0ifAuOcJ442g6GU2s= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -947,8 +951,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20221017184919-83659145692c h1:dveknrit5futqEmXAvd2I1BbZIDhxRijsyWHM86NlcA= -golang.org/x/term v0.0.0-20221017184919-83659145692c/go.mod h1:VTIZ7TEbF0BS9Sv9lPTvGbtW8i4z6GGbJBCM37uMCzY= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1021,8 +1025,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pipeline/backend/common/script.go b/pipeline/backend/common/script.go index e0b94b963..e17d29870 100644 --- a/pipeline/backend/common/script.go +++ b/pipeline/backend/common/script.go @@ -14,18 +14,21 @@ package common -import "runtime" +import ( + "encoding/base64" + "runtime" +) func GenerateContainerConf(commands []string) (env map[string]string, entry, cmd []string) { env = make(map[string]string) if runtime.GOOS == "windows" { - env["CI_SCRIPT"] = generateScriptWindows(commands) + env["CI_SCRIPT"] = base64.StdEncoding.EncodeToString([]byte(generateScriptWindows(commands))) env["HOME"] = "c:\\root" env["SHELL"] = "powershell.exe" entry = []string{"powershell", "-noprofile", "-noninteractive", "-command"} cmd = []string{"[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Env:CI_SCRIPT)) | iex"} } else { - env["CI_SCRIPT"] = generateScriptPosix(commands) + env["CI_SCRIPT"] = base64.StdEncoding.EncodeToString([]byte(generateScriptPosix(commands))) env["HOME"] = "/root" env["SHELL"] = "/bin/sh" entry = []string{"/bin/sh", "-c"} diff --git a/pipeline/backend/common/script_posix.go b/pipeline/backend/common/script_posix.go index 99625223f..e8d21beb1 100644 --- a/pipeline/backend/common/script_posix.go +++ b/pipeline/backend/common/script_posix.go @@ -16,9 +16,9 @@ package common import ( "bytes" - "encoding/base64" "fmt" - "strings" + + "github.com/alessio/shellescape" ) // generateScriptPosix is a helper function that generates a step script @@ -26,11 +26,9 @@ import ( func generateScriptPosix(commands []string) string { var buf bytes.Buffer for _, command := range commands { - escaped := fmt.Sprintf("%q", command) - escaped = strings.Replace(escaped, "$", `\$`, -1) buf.WriteString(fmt.Sprintf( traceScript, - escaped, + shellescape.Quote(command), command, )) } @@ -38,7 +36,7 @@ func generateScriptPosix(commands []string) string { setupScript, buf.String(), ) - return base64.StdEncoding.EncodeToString([]byte(script)) + return script } // setupScript is a helper script this is added to the step script to ensure diff --git a/pipeline/backend/common/script_posix_test.go b/pipeline/backend/common/script_posix_test.go index 5509f71cb..0129666d4 100644 --- a/pipeline/backend/common/script_posix_test.go +++ b/pipeline/backend/common/script_posix_test.go @@ -15,7 +15,6 @@ package common import ( - "encoding/base64" "testing" "github.com/stretchr/testify/assert" @@ -41,13 +40,13 @@ unset CI_NETRC_USERNAME unset CI_NETRC_PASSWORD unset CI_SCRIPT -echo + "echo \${PATH}" +echo + 'echo ${PATH}' echo ${PATH} -echo + "go build" +echo + 'go build' go build -echo + "go test" +echo + 'go test' go test `, @@ -55,8 +54,6 @@ go test } for _, test := range testdata { script := generateScriptPosix(test.from) - decoded, _ := base64.StdEncoding.DecodeString(script) - got := string(decoded) - assert.EqualValues(t, got, test.want, "Want encoded script for %s", test.from) + assert.EqualValues(t, test.want, script, "Want encoded script for %s", test.from) } } diff --git a/pipeline/backend/common/script_win.go b/pipeline/backend/common/script_win.go index 6a6f8c432..b311b0acb 100644 --- a/pipeline/backend/common/script_win.go +++ b/pipeline/backend/common/script_win.go @@ -16,7 +16,6 @@ package common import ( "bytes" - "encoding/base64" "fmt" "strings" ) @@ -36,7 +35,7 @@ func generateScriptWindows(commands []string) string { setupScriptWin, buf.String(), ) - return base64.StdEncoding.EncodeToString([]byte(script)) + return script } const setupScriptWin = ` diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index 51663fb80..7d20e5a48 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -16,18 +16,31 @@ package local import ( "context" - "encoding/base64" + "fmt" "io" "os" "os/exec" "strings" - "github.com/woodpecker-ci/woodpecker/pipeline/backend/common" + "github.com/alessio/shellescape" + "golang.org/x/exp/slices" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" "github.com/woodpecker-ci/woodpecker/shared/constant" ) +// notAllowedEnvVarOverwrites are all env vars that can not be overwritten by step config +var notAllowedEnvVarOverwrites = []string{ + "CI_NETRC_MACHINE", + "CI_NETRC_USERNAME", + "CI_NETRC_PASSWORD", + "CI_SCRIPT", + "HOME", + "SHELL", +} + type local struct { + // TODO: make cmd a cmd list to itterate over, the hard part is to have a common ReadCloser cmd *exec.Cmd output io.ReadCloser workingdir string @@ -65,7 +78,8 @@ func (e *local) Exec(ctx context.Context, step *types.Step) error { // Get environment variables env := os.Environ() for a, b := range step.Environment { - if a != "HOME" && a != "SHELL" { // Don't override $HOME and $SHELL + // append allowed env vars to command env + if !slices.Contains(notAllowedEnvVarOverwrites, a) { env = append(env, a+"="+b) } } @@ -73,6 +87,8 @@ func (e *local) Exec(ctx context.Context, step *types.Step) error { var command []string if step.Image == constant.DefaultCloneImage { // Default clone step + // TODO: creat tmp HOME and insert netrc + // TODO: download plugin-git binary if not exist env = append(env, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"]) command = append(command, "plugin-git") } else { @@ -81,10 +97,14 @@ func (e *local) Exec(ctx context.Context, step *types.Step) error { command = append(command, "-c") // TODO: use commands directly - script, _ := base64.StdEncoding.DecodeString(common.GenerateScript(step.Commands)) - scriptStr := string(script) + script := "" + for _, cmd := range step.Commands { + script += fmt.Sprintf("echo + %s\n%s\n\n", shellescape.Quote(cmd), cmd) + } + script = strings.TrimSpace(script) + // Deleting the initial lines removes netrc support but adds compatibility for more shells like fish - command = append(command, scriptStr[strings.Index(scriptStr, "\n\n")+2:]) + command = append(command, script) } // Prepare command diff --git a/pipeline/backend/ssh/ssh.go b/pipeline/backend/ssh/ssh.go index 8e76d8fc8..cc9ec4b7d 100644 --- a/pipeline/backend/ssh/ssh.go +++ b/pipeline/backend/ssh/ssh.go @@ -2,7 +2,6 @@ package ssh import ( "context" - "encoding/base64" "fmt" "io" "os" @@ -109,10 +108,9 @@ func (e *ssh) Exec(ctx context.Context, step *types.Step) error { command = append(command, "-c") // TODO: use commands directly - script, _ := base64.StdEncoding.DecodeString(common.GenerateScript(step.Commands)) - scriptStr := string(script) + script := common.GenerateScript(step.Commands) // Deleting the initial lines removes netrc support but adds compatibility for more shells like fish - command = append(command, "cd "+e.workingdir+"/"+step.Environment["CI_REPO"]+" && "+scriptStr[strings.Index(scriptStr, "\n\n")+2:]) + command = append(command, "cd "+e.workingdir+"/"+step.Environment["CI_REPO"]+" && "+script[strings.Index(script, "\n\n")+2:]) } // Prepare command