mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-06-10 17:39:37 +00:00
Code Issues Projects Releases Wiki Activity

Ignore pipelines without config (#2949)

Browse source 
Co-authored-by: 6543 <6543@obermui.de>
This commit is contained in:
Anbraten 2023-12-21 11:13:25 +01:00 committed by GitHub
parent 65c62e987f
commit 619858e0e9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 85 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docs/docs/91-migrations.md
View file

@ -5,6 +5,7 @@ Some versions need some changes to the server configuration or the pipeline conf
## `next`
- Removed `WOODPECKER_ROOT_PATH` and `WOODPECKER_ROOT_URL` config variables. Use `WOODPECKER_HOST` with a path instead
- Pipelines without a config file will now be skipped instead of failing
## 2.0.0

2
server/api/helper.go
View file

@ -34,6 +34,8 @@ func handlePipelineErr(c *gin.Context, err error) {
} else if errors.Is(err, &pipeline.ErrBadRequest{}) {
c.String(http.StatusBadRequest, "%s", err)
} else if errors.Is(err, pipeline.ErrFiltered) {
// for debugging purpose we add a header
c.Writer.Header().Add("Pipeline-Filtered", "true")
c.Status(http.StatusNoContent)
} else {
_ = c.AbortWithError(http.StatusInternalServerError, err)

43
server/forge/configFetcher.go
View file

@ -106,7 +106,7 @@ func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config
// could be adapted to allow the user to supply a list like we do in the defaults
configs := []string{config}
fileMeta, err := cf.getFirstAvailableConfig(ctx, configs, true)
fileMeta, err := cf.getFirstAvailableConfig(ctx, configs)
if err == nil {
return fileMeta, err
}
@ -116,7 +116,7 @@ func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config
log.Trace().Msgf("ConfigFetch[%s]: user did not define own config, following default procedure", cf.repo.FullName)
// for the order see shared/constants/constants.go
fileMeta, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:], false)
fileMeta, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:])
if err == nil {
return fileMeta, err
}
@ -141,7 +141,7 @@ func filterPipelineFiles(files []*types.FileMeta) []*types.FileMeta {
return res
}
func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fileMeta []*types.FileMeta, found bool) {
func (cf *configFetcher) checkPipelineFile(c context.Context, config string) ([]*types.FileMeta, error) {
file, err := cf.forge.File(c, cf.user, cf.repo, cf.pipeline, config)
if err == nil && len(file) != 0 {
@ -150,40 +150,47 @@ func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fi
return []*types.FileMeta{{
Name: config,
Data: file,
}}, true
}}, nil
}
return nil, false
return nil, err
}
func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string, userDefined bool) ([]*types.FileMeta, error) {
userDefinedLog := ""
if userDefined {
userDefinedLog = "user defined"
}
func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string) ([]*types.FileMeta, error) {
var forgeErr []error
for _, fileOrFolder := range configs {
if strings.HasSuffix(fileOrFolder, "/") {
// config is a folder
files, err := cf.forge.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/"))
// if folder is not supported we will get a "Not implemented" error and continue
if err != nil && !errors.Is(err, types.ErrNotImplemented) {
log.Error().Err(err).Str("repo", cf.repo.FullName).Str("user", cf.user.Login).Msg("could not get folder from forge")
if err != nil {
if !(errors.Is(err, types.ErrNotImplemented) || errors.Is(err, &types.ErrConfigNotFound{})) {
log.Error().Err(err).Str("repo", cf.repo.FullName).Str("user", cf.user.Login).Msg("could not get folder from forge")
forgeErr = append(forgeErr, err)
}
continue
}
files = filterPipelineFiles(files)
if err == nil && len(files) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found %d %s files in '%s'", cf.repo.FullName, len(files), userDefinedLog, fileOrFolder)
if len(files) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), fileOrFolder)
return files, nil
}
}
// config is a file
if fileMeta, found := cf.checkPipelineFile(c, fileOrFolder); found {
log.Trace().Msgf("ConfigFetch[%s]: found %s file: '%s'", cf.repo.FullName, userDefinedLog, fileOrFolder)
if fileMeta, err := cf.checkPipelineFile(c, fileOrFolder); err == nil {
log.Trace().Msgf("ConfigFetch[%s]: found file: '%s'", cf.repo.FullName, fileOrFolder)
return fileMeta, nil
} else if !errors.Is(err, &types.ErrConfigNotFound{}) {
forgeErr = append(forgeErr, err)
}
}
// got unexpected errors
if len(forgeErr) != 0 {
return nil, errors.Join(forgeErr...)
}
// nothing found
return nil, fmt.Errorf("%s configs not found searched: %s", userDefinedLog, strings.Join(configs, ", "))
return nil, &types.ErrConfigNotFound{Configs: configs}
}

9
server/forge/gitea/gitea.go
View file

@ -39,6 +39,7 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/server"
"go.woodpecker-ci.org/woodpecker/v2/server/forge"
"go.woodpecker-ci.org/woodpecker/v2/server/forge/common"
"go.woodpecker-ci.org/woodpecker/v2/server/forge/types"
forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types"
"go.woodpecker-ci.org/woodpecker/v2/server/model"
"go.woodpecker-ci.org/woodpecker/v2/server/store"
@ -289,7 +290,10 @@ func (c *Gitea) File(ctx context.Context, u *model.User, r *model.Repo, b *model
return nil, err
}
cfg, _, err := client.GetFile(r.Owner, r.Name, b.Commit, f)
cfg, resp, err := client.GetFile(r.Owner, r.Name, b.Commit, f)
if err != nil && resp != nil && resp.StatusCode == http.StatusNotFound {
return nil, errors.Join(err, &types.ErrConfigNotFound{Configs: []string{f}})
}
return cfg, err
}
@ -314,6 +318,9 @@ func (c *Gitea) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model.
if m, _ := filepath.Match(f, e.Path); m && e.Type == "blob" {
data, err := c.File(ctx, u, r, b, e.Path)
if err != nil {
if errors.Is(err, &types.ErrConfigNotFound{}) {
return nil, fmt.Errorf("git tree reported existence of file but we got: %s", err.Error())
}
return nil, fmt.Errorf("multi-pipeline cannot get %s: %w", e.Path, err)
}

15
server/forge/github/github.go
View file

@ -18,6 +18,7 @@ package github
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net/http"
"net/url"
@ -32,6 +33,7 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/server"
"go.woodpecker-ci.org/woodpecker/v2/server/forge"
"go.woodpecker-ci.org/woodpecker/v2/server/forge/common"
"go.woodpecker-ci.org/woodpecker/v2/server/forge/types"
forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types"
"go.woodpecker-ci.org/woodpecker/v2/server/model"
"go.woodpecker-ci.org/woodpecker/v2/server/store"
@ -226,7 +228,10 @@ func (c *client) File(ctx context.Context, u *model.User, r *model.Repo, b *mode
opts := new(github.RepositoryContentGetOptions)
opts.Ref = b.Commit
content, _, _, err := client.Repositories.GetContents(ctx, r.Owner, r.Name, f, opts)
content, _, resp, err := client.Repositories.GetContents(ctx, r.Owner, r.Name, f, opts)
if resp != nil && resp.StatusCode == http.StatusNotFound {
return nil, errors.Join(err, &types.ErrConfigNotFound{Configs: []string{f}})
}
if err != nil {
return nil, err
}
@ -242,7 +247,10 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model
opts := new(github.RepositoryContentGetOptions)
opts.Ref = b.Commit
_, data, _, err := client.Repositories.GetContents(ctx, r.Owner, r.Name, f, opts)
_, data, resp, err := client.Repositories.GetContents(ctx, r.Owner, r.Name, f, opts)
if resp != nil && resp.StatusCode == http.StatusNotFound {
return nil, errors.Join(err, &types.ErrConfigNotFound{Configs: []string{f}})
}
if err != nil {
return nil, err
}
@ -254,6 +262,9 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model
go func(path string) {
content, err := c.File(ctx, u, r, b, path)
if err != nil {
if errors.Is(err, &types.ErrConfigNotFound{}) {
err = fmt.Errorf("git tree reported existence of file but we got: %s", err.Error())
}
errc <- err
} else {
fc <- &forge_types.FileMeta{

10
server/forge/gitlab/gitlab.go
View file

@ -18,6 +18,7 @@ package gitlab
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net/http"
@ -33,6 +34,7 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/server"
"go.woodpecker-ci.org/woodpecker/v2/server/forge"
"go.woodpecker-ci.org/woodpecker/v2/server/forge/common"
"go.woodpecker-ci.org/woodpecker/v2/server/forge/types"
forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types"
"go.woodpecker-ci.org/woodpecker/v2/server/model"
"go.woodpecker-ci.org/woodpecker/v2/server/store"
@ -351,7 +353,10 @@ func (g *GitLab) File(ctx context.Context, user *model.User, repo *model.Repo, p
if err != nil {
return nil, err
}
file, _, err := client.RepositoryFiles.GetRawFile(_repo.ID, fileName, &gitlab.GetRawFileOptions{Ref: &pipeline.Commit}, gitlab.WithContext(ctx))
file, resp, err := client.RepositoryFiles.GetRawFile(_repo.ID, fileName, &gitlab.GetRawFileOptions{Ref: &pipeline.Commit}, gitlab.WithContext(ctx))
if resp != nil && resp.StatusCode == http.StatusNotFound {
return nil, errors.Join(err, &types.ErrConfigNotFound{Configs: []string{fileName}})
}
return file, err
}
@ -388,6 +393,9 @@ func (g *GitLab) Dir(ctx context.Context, user *model.User, repo *model.Repo, pi
}
data, err := g.File(ctx, user, repo, pipeline, batch[i].Path)
if err != nil {
if errors.Is(err, &types.ErrConfigNotFound{}) {
return nil, fmt.Errorf("git tree reported existence of file but we got: %s", err.Error())
}
return nil, err
}
files = append(files, &forge_types.FileMeta{

14
server/forge/types/errors.go
View file

@ -18,6 +18,7 @@ package types
import (
"errors"
"fmt"
"strings"
)
// AuthError represents forge authentication error.
@ -56,3 +57,16 @@ func (*ErrIgnoreEvent) Is(target error) bool {
_, ok := target.(*ErrIgnoreEvent) //nolint:errorlint
return ok
}
type ErrConfigNotFound struct {
Configs []string
}
func (m *ErrConfigNotFound) Error() string {
return fmt.Sprintf("configs not found: %s", strings.Join(m.Configs, ", "))
}
func (*ErrConfigNotFound) Is(target error) bool {
_, ok := target.(*ErrConfigNotFound) //nolint:errorlint
return ok
}

18
server/pipeline/create.go
View file

@ -16,14 +16,16 @@ package pipeline
import (
"context"
"errors"
"fmt"
"regexp"
"github.com/rs/zerolog/log"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/errors"
pipeline_errors "go.woodpecker-ci.org/woodpecker/v2/pipeline/errors"
"go.woodpecker-ci.org/woodpecker/v2/server"
"go.woodpecker-ci.org/woodpecker/v2/server/forge"
forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types"
"go.woodpecker-ci.org/woodpecker/v2/server/model"
"go.woodpecker-ci.org/woodpecker/v2/server/store"
)
@ -64,18 +66,24 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline
// fetch the pipeline file from the forge
configFetcher := forge.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.Timeout, server.Config.Services.ConfigService, repoUser, repo, pipeline)
forgeYamlConfigs, configFetchErr := configFetcher.Fetch(ctx)
if configFetchErr != nil {
if errors.Is(configFetchErr, &forge_types.ErrConfigNotFound{}) {
log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login)
if err := _store.DeletePipeline(pipeline); err != nil {
log.Error().Str("repo", repo.FullName).Err(err).Msg("failed to delete pipeline without config")
}
return nil, ErrFiltered
} else if configFetchErr != nil {
log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("error while fetching config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login)
return nil, updatePipelineWithErr(ctx, _store, pipeline, repo, repoUser, fmt.Errorf("pipeline definition not found in %s", repo.FullName))
}
pipelineItems, parseErr := parsePipeline(_store, pipeline, repoUser, repo, forgeYamlConfigs, nil)
if errors.HasBlockingErrors(parseErr) {
if pipeline_errors.HasBlockingErrors(parseErr) {
log.Debug().Str("repo", repo.FullName).Err(parseErr).Msg("failed to parse yaml")
return nil, updatePipelineWithErr(ctx, _store, pipeline, repo, repoUser, parseErr)
} else if parseErr != nil {
pipeline.Errors = errors.GetPipelineErrors(parseErr)
pipeline.Errors = pipeline_errors.GetPipelineErrors(parseErr)
}
if len(pipelineItems) == 0 {