diff --git a/cli/common/pipeline.go b/cli/common/pipeline.go index c66f58e53..e274f47a5 100644 --- a/cli/common/pipeline.go +++ b/cli/common/pipeline.go @@ -3,26 +3,23 @@ package common import ( "fmt" "os" + "strings" "github.com/urfave/cli/v2" + + "github.com/woodpecker-ci/woodpecker/shared/constant" ) -func DetectPipelineConfig() (multiplies bool, config string, _ error) { - config = ".woodpecker" - if fi, err := os.Stat(config); err == nil && fi.IsDir() { - return true, config, nil +func DetectPipelineConfig() (isDir bool, config string, _ error) { + for _, config := range constant.DefaultConfigOrder { + shouldBeDir := strings.HasSuffix(config, "/") + config = strings.TrimSuffix(config, "/") + + if fi, err := os.Stat(config); err == nil && shouldBeDir == fi.IsDir() { + return fi.IsDir(), config, nil + } } - config = ".woodpecker.yml" - if fi, err := os.Stat(config); err == nil && !fi.IsDir() { - return true, config, nil - } - - config = ".drone.yml" - fi, err := os.Stat(config) - if err == nil && !fi.IsDir() { - return false, config, nil - } return false, "", fmt.Errorf("could not detect pipeline config") } diff --git a/docs/docs/20-usage/25-multi-pipeline.md b/docs/docs/20-usage/25-multi-pipeline.md index 7358d7e39..539b92f7a 100644 --- a/docs/docs/20-usage/25-multi-pipeline.md +++ b/docs/docs/20-usage/25-multi-pipeline.md @@ -6,7 +6,7 @@ This Feature is only available for GitHub, Gitea & GitLab repositories. Follow [ By default, Woodpecker looks for the pipeline definition in `.woodpecker.yml` in the project root. -The Multi-Pipeline feature allows the pipeline to be split into several files and placed in the `.woodpecker/` folder. Only `.yml` files will be used and files in any subfolders like `.woodpecker/sub-folder/test.yml` will be ignored. You can set some custom path like `.my-ci/pipelines/` instead of `.woodpecker/` in the [project settings](./71-project-settings.md). +The Multi-Pipeline feature allows the pipeline to be split into several files and placed in the `.woodpecker/` folder. Only `.yml` and `.yaml` files will be used and files in any subfolders like `.woodpecker/sub-folder/test.yml` will be ignored. You can set some custom path like `.my-ci/pipelines/` instead of `.woodpecker/` in the [project settings](./71-project-settings.md). ## Rational @@ -90,7 +90,7 @@ The pipelines run in parallel on separate agents and share nothing. Dependencies between pipelines can be set with the `depends_on` element. A pipeline doesn't execute until all of its dependencies finished successfully. -The name for a `depends_on` entry is the filename without the path, leading dots and without the file extension `.yml`. If the project config for example uses `.woodpecker/` as path for CI files with a file named `.woodpecker/.lint.yml` the corresponding `depends_on` entry would be `lint`. +The name for a `depends_on` entry is the filename without the path, leading dots and without the file extension `.yml` or `.yaml`. If the project config for example uses `.woodpecker/` as path for CI files with a file named `.woodpecker/.lint.yml` the corresponding `depends_on` entry would be `lint`. ```diff pipeline: diff --git a/docs/docs/20-usage/71-project-settings.md b/docs/docs/20-usage/71-project-settings.md index c2d434da1..b5d579ea8 100644 --- a/docs/docs/20-usage/71-project-settings.md +++ b/docs/docs/20-usage/71-project-settings.md @@ -6,7 +6,7 @@ As the owner of a project in Woodpecker you can change project related settings ## Pipeline path -The path to the pipeline config file or folder. By default it is left empty which will use the following configuration resolution `.woodpecker/*.yml` -> `.woodpecker.yml` -> `.drone.yml`. If you set a custom path Woodpecker tries to load your configuration or fails if no configuration could be found at the specified location. To use a [multi pipeline](./25-multi-pipeline.md) you have to change it to a folder path ending with a `/` like `.woodpecker/`. +The path to the pipeline config file or folder. By default it is left empty which will use the following configuration resolution `.woodpecker/*.yml` and `.woodpecker/*.yaml` (without any preference in handling them) -> `.woodpecker.yml` -> `.woodpecker.yaml` -> `.drone.yml`. If you set a custom path Woodpecker tries to load your configuration or fails if no configuration could be found at the specified location. To use a [multi pipeline](./25-multi-pipeline.md) you have to change it to a folder path ending with a `/` like `.woodpecker/`. ## Repository hooks diff --git a/docs/docs/91-migrations.md b/docs/docs/91-migrations.md index 78700436d..3bd0af8b0 100644 --- a/docs/docs/91-migrations.md +++ b/docs/docs/91-migrations.md @@ -13,6 +13,7 @@ Some versions need some changes to the server configuration or the pipeline conf - Updated Prometheus gauge `build_*` to `pipeline_*` - Updated Prometheus gauge `*_job_*` to `*_step_*` - Renamed config env `WOODPECKER_MAX_PROCS` to `WOODPECKER_MAX_WORKFLOWS` (still available as fallback) +- The pipelines are now also read from `.yaml` files, the new default order is `.woodpecker/*.yml` and `.woodpecker/*.yaml` (without any prioritization) -> `.woodpecker.yml` -> `.woodpecker.yaml` -> `.drone.yml` ## 0.15.0 diff --git a/server/shared/configFetcher.go b/server/shared/configFetcher.go index 8a6de974a..273984f2f 100644 --- a/server/shared/configFetcher.go +++ b/server/shared/configFetcher.go @@ -26,6 +26,7 @@ import ( "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/remote" + "github.com/woodpecker-ci/woodpecker/shared/constant" ) type ConfigFetcher interface { @@ -91,73 +92,36 @@ func (cf *configFetcher) Fetch(ctx context.Context) (files []*remote.FileMeta, e } // fetch config by timeout -// TODO: deduplicate code func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config string) ([]*remote.FileMeta, error) { ctx, cancel := context.WithTimeout(c, timeout) defer cancel() if len(config) > 0 { log.Trace().Msgf("ConfigFetch[%s]: use user config '%s'", cf.repo.FullName, config) - // either a file - if !strings.HasSuffix(config, "/") { - file, err := cf.remote.File(ctx, cf.user, cf.repo, cf.pipeline, config) - if err == nil && len(file) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config) - return []*remote.FileMeta{{ - Name: config, - Data: file, - }}, nil - } + + // could be adapted to allow the user to supply a list like we do in the defaults + configs := []string{config} + + fileMeta, err := cf.getFirstAvailableConfig(ctx, configs, true) + if err == nil { + return fileMeta, err } - // or a folder - files, err := cf.remote.Dir(ctx, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(config, "/")) - if err == nil && len(files) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), config) - return filterPipelineFiles(files), nil - } - - return nil, fmt.Errorf("config '%s' not found: %s", config, err) + return nil, fmt.Errorf("user defined config '%s' not found: %s", config, err) } - log.Trace().Msgf("ConfigFetch[%s]: user did not defined own config follow default procedure", cf.repo.FullName) - // no user defined config so try .woodpecker/*.yml -> .woodpecker.yml -> .drone.yml - - // test .woodpecker/ folder - // if folder is not supported we will get a "Not implemented" error and continue - config = ".woodpecker" - files, err := cf.remote.Dir(ctx, cf.user, cf.repo, cf.pipeline, config) - files = filterPipelineFiles(files) - if err == nil && len(files) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), config) - return files, nil - } - - config = ".woodpecker.yml" - file, err := cf.remote.File(ctx, cf.user, cf.repo, cf.pipeline, config) - if err == nil && len(file) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config) - return []*remote.FileMeta{{ - Name: config, - Data: file, - }}, nil - } - - config = ".drone.yml" - file, err = cf.remote.File(ctx, cf.user, cf.repo, cf.pipeline, config) - if err == nil && len(file) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config) - return []*remote.FileMeta{{ - Name: config, - Data: file, - }}, nil + log.Trace().Msgf("ConfigFetch[%s]: user did not defined own config, following default procedure", cf.repo.FullName) + // for the order see shared/constants/constants.go + fileMeta, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:], false) + if err == nil { + return fileMeta, err } select { case <-ctx.Done(): return nil, ctx.Err() default: - return []*remote.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not found config: %s", err) + return []*remote.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not find config: %s", err) } } @@ -165,10 +129,54 @@ func filterPipelineFiles(files []*remote.FileMeta) []*remote.FileMeta { var res []*remote.FileMeta for _, file := range files { - if strings.HasSuffix(file.Name, ".yml") { + if strings.HasSuffix(file.Name, ".yml") || strings.HasSuffix(file.Name, ".yaml") { res = append(res, file) } } return res } + +func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fileMeta []*remote.FileMeta, found bool) { + file, err := cf.remote.File(c, cf.user, cf.repo, cf.pipeline, config) + + if err == nil && len(file) != 0 { + log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config) + + return []*remote.FileMeta{{ + Name: config, + Data: file, + }}, true + } + + return nil, false +} + +func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string, userDefined bool) ([]*remote.FileMeta, error) { + userDefinedLog := "" + if userDefined { + userDefinedLog = "user defined" + } + + for _, fileOrFolder := range configs { + if strings.HasSuffix(fileOrFolder, "/") { + // config is a folder + // if folder is not supported we will get a "Not implemented" error and continue + files, err := cf.remote.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/")) + files = filterPipelineFiles(files) + if err == nil && len(files) != 0 { + log.Trace().Msgf("ConfigFetch[%s]: found %d %s files in '%s'", cf.repo.FullName, len(files), userDefinedLog, fileOrFolder) + return files, nil + } + } + + // config is a file + if fileMeta, found := cf.checkPipelineFile(c, fileOrFolder); found { + log.Trace().Msgf("ConfigFetch[%s]: found %s file: '%s'", cf.repo.FullName, userDefinedLog, fileOrFolder) + return fileMeta, nil + } + } + + // nothing found + return nil, fmt.Errorf("%s configs not found searched: %s", userDefinedLog, strings.Join(configs, ", ")) +} diff --git a/server/shared/configFetcher_test.go b/server/shared/configFetcher_test.go index 282c98222..7a9fe4383 100644 --- a/server/shared/configFetcher_test.go +++ b/server/shared/configFetcher_test.go @@ -74,6 +74,61 @@ func TestFetch(t *testing.T) { }, expectedError: false, }, + { + name: "Default config with .yaml - .woodpecker/", + repoConfig: "", + files: []file{{ + name: ".woodpecker/text.txt", + data: dummyData, + }, { + name: ".woodpecker/release.yaml", + data: dummyData, + }, { + name: ".woodpecker/image.png", + data: dummyData, + }}, + expectedFileNames: []string{ + ".woodpecker/release.yaml", + }, + expectedError: false, + }, + { + name: "Default config with .yaml, .yml mix - .woodpecker/", + repoConfig: "", + files: []file{{ + name: ".woodpecker/text.txt", + data: dummyData, + }, { + name: ".woodpecker/release.yaml", + data: dummyData, + }, { + name: ".woodpecker/other.yml", + data: dummyData, + }, { + name: ".woodpecker/image.png", + data: dummyData, + }}, + expectedFileNames: []string{ + ".woodpecker/release.yaml", + ".woodpecker/other.yml", + }, + expectedError: false, + }, + { + name: "Default config check .woodpecker.yml before .woodpecker.yaml", + repoConfig: "", + files: []file{{ + name: ".woodpecker.yaml", + data: dummyData, + }, { + name: ".woodpecker.yml", + data: dummyData, + }}, + expectedFileNames: []string{ + ".woodpecker.yml", + }, + expectedError: false, + }, { name: "Override via API with custom config", repoConfig: "", diff --git a/server/shared/stepBuilder.go b/server/shared/stepBuilder.go index 83dee0717..186d9b2cc 100644 --- a/server/shared/stepBuilder.go +++ b/server/shared/stepBuilder.go @@ -400,6 +400,7 @@ func metadataPipelineFromModelPipeline(pipeline *model.Pipeline, includeParent b func SanitizePath(path string) string { path = filepath.Base(path) path = strings.TrimSuffix(path, ".yml") + path = strings.TrimSuffix(path, ".yaml") path = strings.TrimPrefix(path, ".") return path } diff --git a/server/shared/stepBuilder_test.go b/server/shared/stepBuilder_test.go index 2e8662845..e17858e18 100644 --- a/server/shared/stepBuilder_test.go +++ b/server/shared/stepBuilder_test.go @@ -568,6 +568,18 @@ func TestSanitizePath(t *testing.T) { path: "folder/sub-folder/test.yml", sanitizedPath: "test", }, + { + path: ".woodpecker/test.yaml", + sanitizedPath: "test", + }, + { + path: ".woodpecker.yaml", + sanitizedPath: "woodpecker", + }, + { + path: "folder/sub-folder/test.yaml", + sanitizedPath: "test", + }, } for _, test := range testTable { diff --git a/shared/constant/constant.go b/shared/constant/constant.go index 9adfc2ab2..f2bc6637f 100644 --- a/shared/constant/constant.go +++ b/shared/constant/constant.go @@ -22,6 +22,15 @@ var PrivilegedPlugins = []string{ "woodpeckerci/plugin-docker-buildx", } +// DefaultConfigOrder represent the priority in witch woodpecker serarch for a pipeline config by default +// folders are indicated by supplying a trailing / +var DefaultConfigOrder = [...]string{ + ".woodpecker/", + ".woodpecker.yml", + ".woodpecker.yaml", + ".drone.yml", +} + const ( DefaultCloneImage = "docker.io/woodpeckerci/plugin-git:v1.6.0" )