mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-06-02 10:10:31 +00:00
Code Issues Projects Releases Wiki Activity

Fix IPv6 host aliases for kubernetes (#2992)

Browse source 
Closes #2991


[Tests](https://github.com/woodpecker-ci/woodpecker/pull/2993#issuecomment-1868048169)

---------

Co-authored-by: 6543 <6543@obermui.de>
This commit is contained in:
Thomas Anderson 2023-12-23 02:42:30 +03:00 committed by GitHub
parent cd9d425a0d
commit 253d702bc7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 131 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
pipeline/backend/docker/convert.go
View file

@ -88,8 +88,12 @@ func toHostConfig(step *types.Step) *container.HostConfig {
if len(step.DNSSearch) != 0 {
config.DNSSearch = step.DNSSearch
}
extraHosts := []string{}
for _, hostAlias := range step.ExtraHosts {
extraHosts = append(extraHosts, hostAlias.Name+":"+hostAlias.IP)
}
if len(step.ExtraHosts) != 0 {
config.ExtraHosts = step.ExtraHosts
config.ExtraHosts = extraHosts
}
if len(step.Devices) != 0 {
config.Devices = toDev(step.Devices)

10
pipeline/backend/kubernetes/kubernetes.go
View file

@ -20,7 +20,6 @@ import (
"io"
"os"
"runtime"
"strings"
"time"
"github.com/rs/zerolog/log"
@ -166,8 +165,7 @@ func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID s
}
}
extraHosts := []string{}
extraHosts := []types.HostAlias{}
for _, stage := range conf.Stages {
if stage.Alias == "services" {
for _, step := range stage.Steps {
@ -175,12 +173,12 @@ func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID s
if err != nil {
return err
}
extraHosts = append(extraHosts, step.Networks[0].Aliases[0]+":"+svc.Spec.ClusterIP)
hostAlias := types.HostAlias{Name: step.Networks[0].Aliases[0], IP: svc.Spec.ClusterIP}
extraHosts = append(extraHosts, hostAlias)
}
}
}
log.Trace().Msgf("Adding extra hosts: %s", strings.Join(extraHosts, ", "))
log.Trace().Msgf("Adding extra hosts: %v", extraHosts)
for _, stage := range conf.Stages {
for _, step := range stage.Steps {
step.ExtraHosts = extraHosts

23
pipeline/backend/kubernetes/pod.go
View file

@ -36,17 +36,16 @@ const (
func mkPod(namespace, name, image, workDir, goos, serviceAccountName string,
pool, privileged bool,
commands, vols, extraHosts []string,
commands, vols []string,
labels, annotations, env, nodeSelector map[string]string,
tolerations []types.Toleration, resources types.Resources,
extraHosts []types.HostAlias, tolerations []types.Toleration, resources types.Resources,
securityContext *types.SecurityContext, securityContextConfig SecurityContextConfig,
) (*v1.Pod, error) {
var err error
meta := podMeta(name, namespace, labels, annotations)
spec, err := podSpec(serviceAccountName, vols, extraHosts, env,
nodeSelector, tolerations, securityContext, securityContextConfig)
spec, err := podSpec(serviceAccountName, vols, env, nodeSelector, extraHosts, tolerations, securityContext, securityContextConfig)
if err != nil {
return nil, err
}
@ -86,7 +85,8 @@ func podMeta(name, namespace string, labels, annotations map[string]string) meta
return meta
}
func podSpec(serviceAccountName string, vols, extraHosts []string, env, backendNodeSelector map[string]string, backendTolerations []types.Toleration,
func podSpec(serviceAccountName string, vols []string, env, backendNodeSelector map[string]string,
extraHosts []types.HostAlias, backendTolerations []types.Toleration,
securityContext *types.SecurityContext, securityContextConfig SecurityContextConfig,
) (v1.PodSpec, error) {
var err error
@ -195,7 +195,7 @@ func volumeMount(name, path string) v1.VolumeMount {
}
// Here is the service IPs (placed in /etc/hosts in the Pod)
func hostAliases(extraHosts []string) []v1.HostAlias {
func hostAliases(extraHosts []types.HostAlias) []v1.HostAlias {
hostAliases := []v1.HostAlias{}
for _, extraHost := range extraHosts {
hostAlias := hostAlias(extraHost)
@ -204,11 +204,10 @@ func hostAliases(extraHosts []string) []v1.HostAlias {
return hostAliases
}
func hostAlias(extraHost string) v1.HostAlias {
host := strings.Split(extraHost, ":")
func hostAlias(extraHost types.HostAlias) v1.HostAlias {
return v1.HostAlias{
IP: host[1],
Hostnames: []string{host[0]},
IP: extraHost.IP,
Hostnames: []string{extraHost.Name},
}
}
@ -358,9 +357,9 @@ func startPod(ctx context.Context, engine *kube, step *types.Step) (*v1.Pod, err
pod, err := mkPod(engine.config.Namespace, podName, step.Image, step.WorkingDir, engine.goos, step.BackendOptions.Kubernetes.ServiceAccountName,
step.Pull, step.Privileged,
step.Commands, step.Volumes, step.ExtraHosts,
step.Commands, step.Volumes,
engine.config.PodLabels, engine.config.PodAnnotations, step.Environment, step.BackendOptions.Kubernetes.NodeSelector,
step.BackendOptions.Kubernetes.Tolerations, step.BackendOptions.Kubernetes.Resources, step.BackendOptions.Kubernetes.SecurityContext, engine.config.SecurityContext)
step.ExtraHosts, step.BackendOptions.Kubernetes.Tolerations, step.BackendOptions.Kubernetes.Resources, step.BackendOptions.Kubernetes.SecurityContext, engine.config.SecurityContext)
if err != nil {
return nil, err
}

18
pipeline/backend/kubernetes/pod_test.go
View file

@ -108,9 +108,9 @@ func TestTinyPod(t *testing.T) {
pod, err := mkPod("woodpecker", "wp-01he8bebctabr3kgk0qj36d2me-0", "gradle:8.4.0-jdk21", "/woodpecker/src", "linux/amd64", "",
false, false,
[]string{"gradle build"}, []string{"workspace:/woodpecker/src"}, nil,
[]string{"gradle build"}, []string{"workspace:/woodpecker/src"},
nil, nil, map[string]string{"CI": "woodpecker"}, nil,
nil,
nil, nil,
types.Resources{Requests: nil, Limits: nil}, nil, SecurityContextConfig{},
)
assert.NoError(t, err)
@ -228,17 +228,27 @@ func TestFullPod(t *testing.T) {
"hostnames": [
"cloudflare"
]
},
{
"ip": "2606:4700:4700::64",
"hostnames": [
"cf.v6"
]
}
]
},
"status": {}
}`
hostAliases := []types.HostAlias{
{Name: "cloudflare", IP: "1.1.1.1"},
{Name: "cf.v6", IP: "2606:4700:4700::64"},
}
pod, err := mkPod("woodpecker", "wp-01he8bebctabr3kgk0qj36d2me-0", "meltwater/drone-cache", "/woodpecker/src", "linux/amd64", "wp-svc-acc",
true, true,
[]string{"go get", "go test"}, []string{"woodpecker-cache:/woodpecker/src/cache"}, []string{"cloudflare:1.1.1.1"},
[]string{"go get", "go test"}, []string{"woodpecker-cache:/woodpecker/src/cache"},
map[string]string{"app": "test"}, map[string]string{"apparmor.security": "runtime/default"}, map[string]string{"CGO": "0"}, map[string]string{"storage": "ssd"},
[]types.Toleration{{Key: "net-port", Value: "100Mbit", Effect: types.TaintEffectNoSchedule}},
hostAliases, []types.Toleration{{Key: "net-port", Value: "100Mbit", Effect: types.TaintEffectNoSchedule}},
types.Resources{Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"}, Limits: map[string]string{"memory": "256Mi", "cpu": "2"}},
&types.SecurityContext{Privileged: newBool(true), RunAsNonRoot: newBool(true), RunAsUser: newInt64(101), RunAsGroup: newInt64(101), FSGroup: newInt64(101)},
SecurityContextConfig{RunAsNonRoot: false},

5
pipeline/backend/types/network.go
View file

@ -18,3 +18,8 @@ package types
type Network struct {
Name string `json:"name,omitempty"`
}
type HostAlias struct {
Name string `json:"name,omitempty"`
IP string `json:"ip,omitempty"`
}

2
pipeline/backend/types/step.go
View file

@ -28,7 +28,7 @@ type Step struct {
Environment map[string]string `json:"environment,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Commands []string `json:"commands,omitempty"`
ExtraHosts []string `json:"extra_hosts,omitempty"`
ExtraHosts []HostAlias `json:"extra_hosts,omitempty"`
Volumes []string `json:"volumes,omitempty"`
Tmpfs []string `json:"tmpfs,omitempty"`
Devices []string `json:"devices,omitempty"`

91
pipeline/frontend/yaml/compiler/compiler_test.go
View file

@ -82,14 +82,15 @@ func TestCompilerCompile(t *testing.T) {
Name: "test_clone",
Alias: "clone",
Steps: []*backend_types.Step{{
Name: "test_clone",
Alias: "clone",
Type: backend_types.StepTypeClone,
Image: constant.DefaultCloneImage,
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"clone"}}},
Name: "test_clone",
Alias: "clone",
Type: backend_types.StepTypeClone,
Image: constant.DefaultCloneImage,
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"clone"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}
@ -126,14 +127,15 @@ func TestCompilerCompile(t *testing.T) {
Name: "test_stage_0",
Alias: "dummy",
Steps: []*backend_types.Step{{
Name: "test_step_0",
Alias: "dummy",
Type: backend_types.StepTypePlugin,
Image: "dummy_img",
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"dummy"}}},
Name: "test_step_0",
Alias: "dummy",
Type: backend_types.StepTypePlugin,
Image: "dummy_img",
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"dummy"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}},
},
@ -161,39 +163,42 @@ func TestCompilerCompile(t *testing.T) {
Name: "test_stage_0",
Alias: "echo env",
Steps: []*backend_types.Step{{
Name: "test_step_0",
Alias: "echo env",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
Name: "test_step_0",
Alias: "echo env",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}, {
Name: "test_stage_1",
Alias: "parallel echo 1",
Steps: []*backend_types.Step{{
Name: "test_step_1",
Alias: "parallel echo 1",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 1"}}},
Name: "test_step_1",
Alias: "parallel echo 1",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 1"}}},
ExtraHosts: []backend_types.HostAlias{},
}, {
Name: "test_step_2",
Alias: "parallel echo 2",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 2"}}},
Name: "test_step_2",
Alias: "parallel echo 2",
Type: backend_types.StepTypeCommands,
Image: "bash",
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":"},
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 2"}}},
ExtraHosts: []backend_types.HostAlias{},
}},
}},
},

12
pipeline/frontend/yaml/compiler/convert.go
View file

@ -55,6 +55,16 @@ func (c *Compiler) createProcess(name string, container *yaml_types.Container, s
})
}
extraHosts := make([]backend_types.HostAlias, len(container.ExtraHosts))
for i, extraHost := range container.ExtraHosts {
name, ip, ok := strings.Cut(extraHost, ":")
if !ok {
return nil, &ErrExtraHostFormat{host: extraHost}
}
extraHosts[i].Name = name
extraHosts[i].IP = ip
}
var volumes []string
if !c.local {
volumes = append(volumes, workspace)
@ -173,7 +183,7 @@ func (c *Compiler) createProcess(name string, container *yaml_types.Container, s
WorkingDir: workingdir,
Environment: environment,
Commands: container.Commands,
ExtraHosts: container.ExtraHosts,
ExtraHosts: extraHosts,
Volumes: volumes,
Tmpfs: container.Tmpfs,
Devices: container.Devices,

30
pipeline/frontend/yaml/compiler/errors.go Normal file
View file

@ -0,0 +1,30 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compiler
import "fmt"
type ErrExtraHostFormat struct {
host string
}
func (err *ErrExtraHostFormat) Error() string {
return fmt.Sprintf("extra host %s is in wrong format", err.host)
}
func (*ErrExtraHostFormat) Is(target error) bool {
_, ok := target.(*ErrExtraHostFormat) //nolint:errorlint
return ok
}

3
pipeline/frontend/yaml/types/container_test.go
View file

@ -45,6 +45,7 @@ environment:
extra_hosts:
- somehost:162.242.195.82
- otherhost:50.31.209.229
- ipv6:2001:db8::10
name: my-build-container
network_mode: bridge
networks:
@ -84,7 +85,7 @@ func TestUnmarshalContainer(t *testing.T) {
DNS: base.StringOrSlice{"8.8.8.8"},
DNSSearch: base.StringOrSlice{"example.com"},
Environment: base.SliceOrMap{"RACK_ENV": "development", "SHOW": "true"},
ExtraHosts: []string{"somehost:162.242.195.82", "otherhost:50.31.209.229"},
ExtraHosts: []string{"somehost:162.242.195.82", "otherhost:50.31.209.229", "ipv6:2001:db8::10"},
Image: "golang:latest",
MemLimit: base.MemStringOrInt(1024),
MemSwapLimit: base.MemStringOrInt(1024),