From 2477d2e57f77726a0ceb10bf3b4baafaee8a1556 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 3 Nov 2022 19:12:40 +0100 Subject: [PATCH] Support .yaml as file-ending for workflow config too (#1388) This implements #1073, adds .yaml to the accepted endings for woodpecker configs. This currently adds some more lines to the duplication (tried to compensate by fixing the other duplication in the configFetcher) as the CLI and Server are still separate. --- cli/common/pipeline.go | 25 +++-- docs/docs/20-usage/25-multi-pipeline.md | 4 +- docs/docs/20-usage/71-project-settings.md | 2 +- docs/docs/91-migrations.md | 1 + server/shared/configFetcher.go | 112 ++++++++++++---------- server/shared/configFetcher_test.go | 55 +++++++++++ server/shared/stepBuilder.go | 1 + server/shared/stepBuilder_test.go | 12 +++ shared/constant/constant.go | 9 ++ 9 files changed, 152 insertions(+), 69 deletions(-) diff --git a/cli/common/pipeline.go b/cli/common/pipeline.go index c66f58e53..e274f47a5 100644 --- a/cli/common/pipeline.go +++ b/cli/common/pipeline.go @@ -3,26 +3,23 @@ package common import ( "fmt" "os" + "strings" "github.com/urfave/cli/v2" + + "github.com/woodpecker-ci/woodpecker/shared/constant" ) -func DetectPipelineConfig() (multiplies bool, config string, _ error) { - config = ".woodpecker" - if fi, err := os.Stat(config); err == nil && fi.IsDir() { - return true, config, nil +func DetectPipelineConfig() (isDir bool, config string, _ error) { + for _, config := range constant.DefaultConfigOrder { + shouldBeDir := strings.HasSuffix(config, "/") + config = strings.TrimSuffix(config, "/") + + if fi, err := os.Stat(config); err == nil && shouldBeDir == fi.IsDir() { + return fi.IsDir(), config, nil + } } - config = ".woodpecker.yml" - if fi, err := os.Stat(config); err == nil && !fi.IsDir() { - return true, config, nil - } - - config = ".drone.yml" - fi, err := os.Stat(config) - if err == nil && !fi.IsDir() { - return false, config, nil - } return false, "", fmt.Errorf("could not detect pipeline config") } diff --git a/docs/docs/20-usage/25-multi-pipeline.md b/docs/docs/20-usage/25-multi-pipeline.md index 7358d7e39..539b92f7a 100644 --- a/docs/docs/20-usage/25-multi-pipeline.md +++ b/docs/docs/20-usage/25-multi-pipeline.md @@ -6,7 +6,7 @@ This Feature is only available for GitHub, Gitea & GitLab repositories. Follow [ By default, Woodpecker looks for the pipeline definition in `.woodpecker.yml` in the project root. -The Multi-Pipeline feature allows the pipeline to be split into several files and placed in the `.woodpecker/` folder. Only `.yml` files will be used and files in any subfolders like `.woodpecker/sub-folder/test.yml` will be ignored. You can set some custom path like `.my-ci/pipelines/` instead of `.woodpecker/` in the [project settings](./71-project-settings.md). +The Multi-Pipeline feature allows the pipeline to be split into several files and placed in the `.woodpecker/` folder. Only `.yml` and `.yaml` files will be used and files in any subfolders like `.woodpecker/sub-folder/test.yml` will be ignored. You can set some custom path like `.my-ci/pipelines/` instead of `.woodpecker/` in the [project settings](./71-project-settings.md). ## Rational @@ -90,7 +90,7 @@ The pipelines run in parallel on separate agents and share nothing. Dependencies between pipelines can be set with the `depends_on` element. A pipeline doesn't execute until all of its dependencies finished successfully. -The name for a `depends_on` entry is the filename without the path, leading dots and without the file extension `.yml`. If the project config for example uses `.woodpecker/` as path for CI files with a file named `.woodpecker/.lint.yml` the corresponding `depends_on` entry would be `lint`. +The name for a `depends_on` entry is the filename without the path, leading dots and without the file extension `.yml` or `.yaml`. If the project config for example uses `.woodpecker/` as path for CI files with a file named `.woodpecker/.lint.yml` the corresponding `depends_on` entry would be `lint`. ```diff pipeline: diff --git a/docs/docs/20-usage/71-project-settings.md b/docs/docs/20-usage/71-project-settings.md index c2d434da1..b5d579ea8 100644 --- a/docs/docs/20-usage/71-project-settings.md +++ b/docs/docs/20-usage/71-project-settings.md @@ -6,7 +6,7 @@ As the owner of a project in Woodpecker you can change project related settings ## Pipeline path -The path to the pipeline config file or folder. By default it is left empty which will use the following configuration resolution `.woodpecker/*.yml` -> `.woodpecker.yml` -> `.drone.yml`. If you set a custom path Woodpecker tries to load your configuration or fails if no configuration could be found at the specified location. To use a [multi pipeline](./25-multi-pipeline.md) you have to change it to a folder path ending with a `/` like `.woodpecker/`. +The path to the pipeline config file or folder. By default it is left empty which will use the following configuration resolution `.woodpecker/*.yml` and `.woodpecker/*.yaml` (without any preference in handling them) -> `.woodpecker.yml` -> `.woodpecker.yaml` -> `.drone.yml`. If you set a custom path Woodpecker tries to load your configuration or fails if no configuration could be found at the specified location. To use a [multi pipeline](./25-multi-pipeline.md) you have to change it to a folder path ending with a `/` like `.woodpecker/`. ## Repository hooks diff --git a/docs/docs/91-migrations.md b/docs/docs/91-migrations.md index 78700436d..3bd0af8b0 100644 --- a/docs/docs/91-migrations.md +++ b/docs/docs/91-migrations.md @@ -13,6 +13,7 @@ Some versions need some changes to the server configuration or the pipeline conf - Updated Prometheus gauge `build_*` to `pipeline_*` - Updated Prometheus gauge `*_job_*` to `*_step_*` - Renamed config env `WOODPECKER_MAX_PROCS` to `WOODPECKER_MAX_WORKFLOWS` (still available as fallback) +- The pipelines are now also read from `.yaml` files, the new default order is `.woodpecker/*.yml` and `.woodpecker/*.yaml` (without any prioritization) -> `.woodpecker.yml` -> `.woodpecker.yaml` -> `.drone.yml` ## 0.15.0 diff --git a/server/shared/configFetcher.go b/server/shared/configFetcher.go index 8a6de974a..273984f2f 100644 --- a/server/shared/configFetcher.go +++ b/server/shared/configFetcher.go @@ -26,6 +26,7 @@ import ( "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/remote" + "github.com/woodpecker-ci/woodpecker/shared/constant" ) type ConfigFetcher interface { @@ -91,73 +92,36 @@ func (cf *configFetcher) Fetch(ctx context.Context) (files []*remote.FileMeta, e } // fetch config by timeout -// TODO: deduplicate code func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config string) ([]*remote.FileMeta, error) { ctx, cancel := context.WithTimeout(c, timeout) defer cancel() if len(config) > 0 { log.Trace().Msgf("ConfigFetch[%s]: use user config '%s'", cf.repo.FullName, config) - // either a file - if !strings.HasSuffix(config, "/") { - file, err := cf.remote.File(ctx, cf.user, cf.repo, cf.pipeline, config) - if err == nil && len(file) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config) - return []*remote.FileMeta{{ - Name: config, - Data: file, - }}, nil - } + + // could be adapted to allow the user to supply a list like we do in the defaults + configs := []string{config} + + fileMeta, err := cf.getFirstAvailableConfig(ctx, configs, true) + if err == nil { + return fileMeta, err } - // or a folder - files, err := cf.remote.Dir(ctx, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(config, "/")) - if err == nil && len(files) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), config) - return filterPipelineFiles(files), nil - } - - return nil, fmt.Errorf("config '%s' not found: %s", config, err) + return nil, fmt.Errorf("user defined config '%s' not found: %s", config, err) } - log.Trace().Msgf("ConfigFetch[%s]: user did not defined own config follow default procedure", cf.repo.FullName) - // no user defined config so try .woodpecker/*.yml -> .woodpecker.yml -> .drone.yml - - // test .woodpecker/ folder - // if folder is not supported we will get a "Not implemented" error and continue - config = ".woodpecker" - files, err := cf.remote.Dir(ctx, cf.user, cf.repo, cf.pipeline, config) - files = filterPipelineFiles(files) - if err == nil && len(files) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), config) - return files, nil - } - - config = ".woodpecker.yml" - file, err := cf.remote.File(ctx, cf.user, cf.repo, cf.pipeline, config) - if err == nil && len(file) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config) - return []*remote.FileMeta{{ - Name: config, - Data: file, - }}, nil - } - - config = ".drone.yml" - file, err = cf.remote.File(ctx, cf.user, cf.repo, cf.pipeline, config) - if err == nil && len(file) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config) - return []*remote.FileMeta{{ - Name: config, - Data: file, - }}, nil + log.Trace().Msgf("ConfigFetch[%s]: user did not defined own config, following default procedure", cf.repo.FullName) + // for the order see shared/constants/constants.go + fileMeta, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:], false) + if err == nil { + return fileMeta, err } select { case <-ctx.Done(): return nil, ctx.Err() default: - return []*remote.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not found config: %s", err) + return []*remote.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not find config: %s", err) } } @@ -165,10 +129,54 @@ func filterPipelineFiles(files []*remote.FileMeta) []*remote.FileMeta { var res []*remote.FileMeta for _, file := range files { - if strings.HasSuffix(file.Name, ".yml") { + if strings.HasSuffix(file.Name, ".yml") || strings.HasSuffix(file.Name, ".yaml") { res = append(res, file) } } return res } + +func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fileMeta []*remote.FileMeta, found bool) { + file, err := cf.remote.File(c, cf.user, cf.repo, cf.pipeline, config) + + if err == nil && len(file) != 0 { + log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config) + + return []*remote.FileMeta{{ + Name: config, + Data: file, + }}, true + } + + return nil, false +} + +func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string, userDefined bool) ([]*remote.FileMeta, error) { + userDefinedLog := "" + if userDefined { + userDefinedLog = "user defined" + } + + for _, fileOrFolder := range configs { + if strings.HasSuffix(fileOrFolder, "/") { + // config is a folder + // if folder is not supported we will get a "Not implemented" error and continue + files, err := cf.remote.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/")) + files = filterPipelineFiles(files) + if err == nil && len(files) != 0 { + log.Trace().Msgf("ConfigFetch[%s]: found %d %s files in '%s'", cf.repo.FullName, len(files), userDefinedLog, fileOrFolder) + return files, nil + } + } + + // config is a file + if fileMeta, found := cf.checkPipelineFile(c, fileOrFolder); found { + log.Trace().Msgf("ConfigFetch[%s]: found %s file: '%s'", cf.repo.FullName, userDefinedLog, fileOrFolder) + return fileMeta, nil + } + } + + // nothing found + return nil, fmt.Errorf("%s configs not found searched: %s", userDefinedLog, strings.Join(configs, ", ")) +} diff --git a/server/shared/configFetcher_test.go b/server/shared/configFetcher_test.go index 282c98222..7a9fe4383 100644 --- a/server/shared/configFetcher_test.go +++ b/server/shared/configFetcher_test.go @@ -74,6 +74,61 @@ func TestFetch(t *testing.T) { }, expectedError: false, }, + { + name: "Default config with .yaml - .woodpecker/", + repoConfig: "", + files: []file{{ + name: ".woodpecker/text.txt", + data: dummyData, + }, { + name: ".woodpecker/release.yaml", + data: dummyData, + }, { + name: ".woodpecker/image.png", + data: dummyData, + }}, + expectedFileNames: []string{ + ".woodpecker/release.yaml", + }, + expectedError: false, + }, + { + name: "Default config with .yaml, .yml mix - .woodpecker/", + repoConfig: "", + files: []file{{ + name: ".woodpecker/text.txt", + data: dummyData, + }, { + name: ".woodpecker/release.yaml", + data: dummyData, + }, { + name: ".woodpecker/other.yml", + data: dummyData, + }, { + name: ".woodpecker/image.png", + data: dummyData, + }}, + expectedFileNames: []string{ + ".woodpecker/release.yaml", + ".woodpecker/other.yml", + }, + expectedError: false, + }, + { + name: "Default config check .woodpecker.yml before .woodpecker.yaml", + repoConfig: "", + files: []file{{ + name: ".woodpecker.yaml", + data: dummyData, + }, { + name: ".woodpecker.yml", + data: dummyData, + }}, + expectedFileNames: []string{ + ".woodpecker.yml", + }, + expectedError: false, + }, { name: "Override via API with custom config", repoConfig: "", diff --git a/server/shared/stepBuilder.go b/server/shared/stepBuilder.go index 83dee0717..186d9b2cc 100644 --- a/server/shared/stepBuilder.go +++ b/server/shared/stepBuilder.go @@ -400,6 +400,7 @@ func metadataPipelineFromModelPipeline(pipeline *model.Pipeline, includeParent b func SanitizePath(path string) string { path = filepath.Base(path) path = strings.TrimSuffix(path, ".yml") + path = strings.TrimSuffix(path, ".yaml") path = strings.TrimPrefix(path, ".") return path } diff --git a/server/shared/stepBuilder_test.go b/server/shared/stepBuilder_test.go index 2e8662845..e17858e18 100644 --- a/server/shared/stepBuilder_test.go +++ b/server/shared/stepBuilder_test.go @@ -568,6 +568,18 @@ func TestSanitizePath(t *testing.T) { path: "folder/sub-folder/test.yml", sanitizedPath: "test", }, + { + path: ".woodpecker/test.yaml", + sanitizedPath: "test", + }, + { + path: ".woodpecker.yaml", + sanitizedPath: "woodpecker", + }, + { + path: "folder/sub-folder/test.yaml", + sanitizedPath: "test", + }, } for _, test := range testTable { diff --git a/shared/constant/constant.go b/shared/constant/constant.go index 9adfc2ab2..f2bc6637f 100644 --- a/shared/constant/constant.go +++ b/shared/constant/constant.go @@ -22,6 +22,15 @@ var PrivilegedPlugins = []string{ "woodpeckerci/plugin-docker-buildx", } +// DefaultConfigOrder represent the priority in witch woodpecker serarch for a pipeline config by default +// folders are indicated by supplying a trailing / +var DefaultConfigOrder = [...]string{ + ".woodpecker/", + ".woodpecker.yml", + ".woodpecker.yaml", + ".drone.yml", +} + const ( DefaultCloneImage = "docker.io/woodpeckerci/plugin-git:v1.6.0" )