GenerateScript should not return encoded script (#1397)

followup to #1395
This commit is contained in:
6543 2022-11-06 13:36:34 +01:00 committed by GitHub
parent 18311d4360
commit e8490a757f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 64 additions and 43 deletions

12
go.mod
View File

@ -5,6 +5,7 @@ go 1.18
require (
code.gitea.io/sdk/gitea v0.15.1-0.20221016183512-2d9ee57af1e0
codeberg.org/6543/go-yaml2json v0.3.0
github.com/alessio/shellescape v1.4.1
github.com/antonmedv/expr v1.9.0
github.com/bmatcuk/doublestar/v4 v4.2.0
github.com/caddyserver/certmagic v0.17.2
@ -38,7 +39,8 @@ require (
github.com/urfave/cli/v2 v2.20.2
github.com/xanzy/go-gitlab v0.73.1
github.com/xeipuuv/gojsonschema v1.2.0
golang.org/x/net v0.0.0-20221019024206-cb67ada4b0ad
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
golang.org/x/net v0.1.0
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783
golang.org/x/sync v0.1.0
google.golang.org/grpc v1.50.1
@ -125,13 +127,13 @@ require (
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.23.0 // indirect
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/mod v0.6.0 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/term v0.0.0-20221017184919-83659145692c // indirect
golang.org/x/term v0.1.0 // indirect
golang.org/x/text v0.4.0 // indirect
golang.org/x/time v0.1.0 // indirect
golang.org/x/tools v0.1.12 // indirect
golang.org/x/tools v0.2.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect

24
go.sum
View File

@ -59,6 +59,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/antonmedv/expr v1.9.0 h1:j4HI3NHEdgDnN9p6oI6Ndr0G5QryMY0FNxT4ONrFDGU=
github.com/antonmedv/expr v1.9.0/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8=
@ -766,8 +768,8 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a h1:NmSIgad6KjE6VvHciPZuNRTKxGhlPfD6OA87W/PLkqg=
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -778,6 +780,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE=
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -799,8 +803,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -846,8 +850,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20221019024206-cb67ada4b0ad h1:Zx6wVVDwwNJFWXNIvDi7o952w3/1ckSwYk/7eykRmjM=
golang.org/x/net v0.0.0-20221019024206-cb67ada4b0ad/go.mod h1:RpDiru2p0u2F0lLpEoqnP2+7xs0ifAuOcJ442g6GU2s=
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -947,8 +951,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20221017184919-83659145692c h1:dveknrit5futqEmXAvd2I1BbZIDhxRijsyWHM86NlcA=
golang.org/x/term v0.0.0-20221017184919-83659145692c/go.mod h1:VTIZ7TEbF0BS9Sv9lPTvGbtW8i4z6GGbJBCM37uMCzY=
golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1021,8 +1025,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@ -14,18 +14,21 @@
package common
import "runtime"
import (
"encoding/base64"
"runtime"
)
func GenerateContainerConf(commands []string) (env map[string]string, entry, cmd []string) {
env = make(map[string]string)
if runtime.GOOS == "windows" {
env["CI_SCRIPT"] = generateScriptWindows(commands)
env["CI_SCRIPT"] = base64.StdEncoding.EncodeToString([]byte(generateScriptWindows(commands)))
env["HOME"] = "c:\\root"
env["SHELL"] = "powershell.exe"
entry = []string{"powershell", "-noprofile", "-noninteractive", "-command"}
cmd = []string{"[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Env:CI_SCRIPT)) | iex"}
} else {
env["CI_SCRIPT"] = generateScriptPosix(commands)
env["CI_SCRIPT"] = base64.StdEncoding.EncodeToString([]byte(generateScriptPosix(commands)))
env["HOME"] = "/root"
env["SHELL"] = "/bin/sh"
entry = []string{"/bin/sh", "-c"}

View File

@ -16,9 +16,9 @@ package common
import (
"bytes"
"encoding/base64"
"fmt"
"strings"
"github.com/alessio/shellescape"
)
// generateScriptPosix is a helper function that generates a step script
@ -26,11 +26,9 @@ import (
func generateScriptPosix(commands []string) string {
var buf bytes.Buffer
for _, command := range commands {
escaped := fmt.Sprintf("%q", command)
escaped = strings.Replace(escaped, "$", `\$`, -1)
buf.WriteString(fmt.Sprintf(
traceScript,
escaped,
shellescape.Quote(command),
command,
))
}
@ -38,7 +36,7 @@ func generateScriptPosix(commands []string) string {
setupScript,
buf.String(),
)
return base64.StdEncoding.EncodeToString([]byte(script))
return script
}
// setupScript is a helper script this is added to the step script to ensure

View File

@ -15,7 +15,6 @@
package common
import (
"encoding/base64"
"testing"
"github.com/stretchr/testify/assert"
@ -41,13 +40,13 @@ unset CI_NETRC_USERNAME
unset CI_NETRC_PASSWORD
unset CI_SCRIPT
echo + "echo \${PATH}"
echo + 'echo ${PATH}'
echo ${PATH}
echo + "go build"
echo + 'go build'
go build
echo + "go test"
echo + 'go test'
go test
`,
@ -55,8 +54,6 @@ go test
}
for _, test := range testdata {
script := generateScriptPosix(test.from)
decoded, _ := base64.StdEncoding.DecodeString(script)
got := string(decoded)
assert.EqualValues(t, got, test.want, "Want encoded script for %s", test.from)
assert.EqualValues(t, test.want, script, "Want encoded script for %s", test.from)
}
}

View File

@ -16,7 +16,6 @@ package common
import (
"bytes"
"encoding/base64"
"fmt"
"strings"
)
@ -36,7 +35,7 @@ func generateScriptWindows(commands []string) string {
setupScriptWin,
buf.String(),
)
return base64.StdEncoding.EncodeToString([]byte(script))
return script
}
const setupScriptWin = `

View File

@ -16,18 +16,31 @@ package local
import (
"context"
"encoding/base64"
"fmt"
"io"
"os"
"os/exec"
"strings"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
"github.com/alessio/shellescape"
"golang.org/x/exp/slices"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/shared/constant"
)
// notAllowedEnvVarOverwrites are all env vars that can not be overwritten by step config
var notAllowedEnvVarOverwrites = []string{
"CI_NETRC_MACHINE",
"CI_NETRC_USERNAME",
"CI_NETRC_PASSWORD",
"CI_SCRIPT",
"HOME",
"SHELL",
}
type local struct {
// TODO: make cmd a cmd list to itterate over, the hard part is to have a common ReadCloser
cmd *exec.Cmd
output io.ReadCloser
workingdir string
@ -65,7 +78,8 @@ func (e *local) Exec(ctx context.Context, step *types.Step) error {
// Get environment variables
env := os.Environ()
for a, b := range step.Environment {
if a != "HOME" && a != "SHELL" { // Don't override $HOME and $SHELL
// append allowed env vars to command env
if !slices.Contains(notAllowedEnvVarOverwrites, a) {
env = append(env, a+"="+b)
}
}
@ -73,6 +87,8 @@ func (e *local) Exec(ctx context.Context, step *types.Step) error {
var command []string
if step.Image == constant.DefaultCloneImage {
// Default clone step
// TODO: creat tmp HOME and insert netrc
// TODO: download plugin-git binary if not exist
env = append(env, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"])
command = append(command, "plugin-git")
} else {
@ -81,10 +97,14 @@ func (e *local) Exec(ctx context.Context, step *types.Step) error {
command = append(command, "-c")
// TODO: use commands directly
script, _ := base64.StdEncoding.DecodeString(common.GenerateScript(step.Commands))
scriptStr := string(script)
script := ""
for _, cmd := range step.Commands {
script += fmt.Sprintf("echo + %s\n%s\n\n", shellescape.Quote(cmd), cmd)
}
script = strings.TrimSpace(script)
// Deleting the initial lines removes netrc support but adds compatibility for more shells like fish
command = append(command, scriptStr[strings.Index(scriptStr, "\n\n")+2:])
command = append(command, script)
}
// Prepare command

View File

@ -2,7 +2,6 @@ package ssh
import (
"context"
"encoding/base64"
"fmt"
"io"
"os"
@ -109,10 +108,9 @@ func (e *ssh) Exec(ctx context.Context, step *types.Step) error {
command = append(command, "-c")
// TODO: use commands directly
script, _ := base64.StdEncoding.DecodeString(common.GenerateScript(step.Commands))
scriptStr := string(script)
script := common.GenerateScript(step.Commands)
// Deleting the initial lines removes netrc support but adds compatibility for more shells like fish
command = append(command, "cd "+e.workingdir+"/"+step.Environment["CI_REPO"]+" && "+scriptStr[strings.Index(scriptStr, "\n\n")+2:])
command = append(command, "cd "+e.workingdir+"/"+step.Environment["CI_REPO"]+" && "+script[strings.Index(script, "\n\n")+2:])
}
// Prepare command