Add support for pipeline configuration service (#804)

* Add configuration extension flags to server Add httpsignatures dependency Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add http fetching to config fetcher Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Refetch config on rebuild Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * - Ensure multipipeline compatiblity - Send original config in http request Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Basic tests of config api Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Simple docs page Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Better flag naming Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Rename usages of the term yaml Rename ConfigAPI struct Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Doc adjustments Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * More docs touchups Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix env vars in docs Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * fix json tags for api calls Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add example config service Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Consistent naming for configService Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Docs: Change example repository location Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix tests after response field rename Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Revert accidential unrelated change in api hook Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Update server flag descriptions Co-authored-by: Anbraten <anton@ju60.de> Co-authored-by: Anbraten <anton@ju60.de>
2024-06-03 05:59:30 +00:00 · 2022-02-28 10:56:23 +01:00 · 2022-02-28 10:56:23 +01:00 · 59ba8538a1
parent a3ac393264
commit 59ba8538a1
18 changed files with 886 additions and 27 deletions
--- a/cmd/server/flags.go
+++ b/cmd/server/flags.go
@ -161,6 +161,16 @@ var flags = []cli.Flag{
 		Name:    "gating-service",
 		Usage:   "gated build endpoint",
 	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_CONFIG_SERVICE_ENDPOINT"},
+		Name:    "config-service-endpoint",
+		Usage:   "url used for calling configuration service endpoint",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_CONFIG_SERVICE_SECRET"},
+		Name:    "config-service-secret",
+		Usage:   "secret to sign requests send to configuration service",
+	},
 	&cli.StringFlag{
 		EnvVars: []string{"WOODPECKER_DATABASE_DRIVER"},
 		Name:    "driver",
--- a/cmd/server/server.go
+++ b/cmd/server/server.go
@ -41,6 +41,7 @@ import (
 	"github.com/woodpecker-ci/woodpecker/server"
 	woodpeckerGrpcServer "github.com/woodpecker-ci/woodpecker/server/grpc"
 	"github.com/woodpecker-ci/woodpecker/server/logging"
+	"github.com/woodpecker-ci/woodpecker/server/plugins/configuration"
 	"github.com/woodpecker-ci/woodpecker/server/plugins/sender"
 	"github.com/woodpecker-ci/woodpecker/server/pubsub"
 	"github.com/woodpecker-ci/woodpecker/server/remote"
@ -271,6 +272,15 @@ func setupEvilGlobals(c *cli.Context, v store.Store, r remote.Remote) {
 		server.Config.Services.Senders = sender.NewRemote(endpoint)
 	}

+	if endpoint := c.String("config-service-endpoint"); endpoint != "" {
+		secret := c.String("config-service-secret")
+		if secret == "" {
+			log.Error().Msg("could not configure configuration service, missing secret")
+		} else {
+			server.Config.Services.ConfigService = configuration.NewAPI(endpoint, secret)
+		}
+	}
+
 	// authentication
 	server.Config.Pipeline.AuthenticatePublicRepos = c.Bool("authenticate-public-repos")

--- a/docs/docs/30-administration/100-external-configuration-api.md
+++ b/docs/docs/30-administration/100-external-configuration-api.md
@ -0,0 +1,106 @@
+# External Configuration API
+
+To provide additional management and preprocessing capabilities for pipeline configurations Woodpecker supports an HTTP api which can be enabled to call an external config service.
+Before the run or restart of any pipeline Woodpecker will make a POST request to an external HTTP api sending the current repository, build information and all current config files retrieved from the repository. The external api can then send back new pipeline configurations that will be used immediately or respond with `HTTP 204` to tell the system to use the existing configuration.
+
+Every request sent by Woodpecker is signed using a http-signature using the provided secret from `WOODPECKER_CONFIG_SERVICE_SECRET`. This way the external api can verify the authenticity request from the Woodpecker instance.
+
+A simplistic example configuration service can be found here: [https://github.com/woodpecker-ci/example-config-service](https://github.com/woodpecker-ci/example-config-service)
+
+## Config
+
+```shell
+# Server
+# ...
+WOODPECKER_CONFIG_SERVICE_ENDPOINT=https://example.com/ciconfig
+WOODPECKER_CONFIG_SERVICE_SECRET=mysecretsigningkey
+
+```
+
+### Example request made by Woodpecker
+
+```json
+{
+  "repo": {
+    "id": 100,
+    "uid": "",
+    "user_id": 0,
+    "namespace": "",
+    "name": "woodpecker-testpipe",
+    "slug": "",
+    "scm": "git",
+    "git_http_url": "",
+    "git_ssh_url": "",
+    "link": "",
+    "default_branhc": "",
+    "private": true,
+    "visibility": "private",
+    "active": true,
+    "config": "",
+    "trusted": false,
+    "protected": false,
+    "ignore_forks": false,
+    "ignore_pulls": false,
+    "cancel_pulls": false,
+    "timeout": 60,
+    "counter": 0,
+    "synced": 0,
+    "created": 0,
+    "updated": 0,
+    "version": 0
+  },
+  "build": {
+    "author": "myUser",
+    "author_avatar": "https://myscm.com/avatars/d6b3f7787a685fcdf2a44e2c685c7e03",
+    "author_email": "my@email.com",
+    "branch": "master",
+    "changed_files": [
+      "somefilename.txt"
+    ],
+    "commit": "2fff90f8d288a4640e90f05049fe30e61a14fd50",
+    "created_at": 0,
+    "deploy_to": "",
+    "enqueued_at": 0,
+    "error": "",
+    "event": "push",
+    "finished_at": 0,
+    "id": 0,
+    "link_url": "https://myscm.com/myUser/woodpecker-testpipe/commit/2fff90f8d288a4640e90f05049fe30e61a14fd50",
+    "message": "test old config\n",
+    "number": 0,
+    "parent": 0,
+    "ref": "refs/heads/master",
+    "refspec": "",
+    "remote": "",
+    "reviewed_at": 0,
+    "reviewed_by": "",
+    "sender": "myUser",
+    "signed": false,
+    "started_at": 0,
+    "status": "",
+    "timestamp": 1645962783,
+    "title": "",
+    "updated_at": 0,
+    "verified": false
+  },
+  "config": [
+    {
+      "name": ".woodpecekr.yml",
+      "data": "pipeline:\n  backend:\n    image: alpine\n    commands:\n      - echo \"Hello there from Repo (.woodpecekr.yml)\"\n"
+    }
+  ]
+}
+```
+
+### Example response structure
+
+```json
+{
+  "pipelines": [
+    {
+      "name": "central-override",
+      "data": "pipeline:\n  backend:\n    image: alpine\n    commands:\n      - echo \"Hello there from ConfigAPI\"\n"
+    }
+  ]
+}
+```
--- a/go.mod
+++ b/go.mod
@ -4,6 +4,7 @@ go 1.16

 require (
 	code.gitea.io/sdk/gitea v0.15.0
+	github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e
 	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.0.2
 	github.com/containerd/containerd v1.5.9 // indirect
--- a/go.sum
+++ b/go.sum
@ -60,6 +60,8 @@ contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EU
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=
+github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e h1:rl2Aq4ZODqTDkeSqQBy+fzpZPamacO1Srp8zq7jf2Sc=
+github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e/go.mod h1:Xa6lInWHNQnuWoF0YPSsx+INFA9qk7/7pTjwb3PInkY=
 github.com/Antonboom/errname v0.1.5 h1:IM+A/gz0pDhKmlt5KSNTVAvfLMb+65RxavBXpRtCUEg=
 github.com/Antonboom/errname v0.1.5/go.mod h1:DugbBstvPFQbv/5uLcRRzfrNqKE9tVdVCqWCLp6Cifo=
 github.com/Antonboom/nilnil v0.1.0 h1:DLDavmg0a6G/F4Lt9t7Enrbgb3Oph6LnDE6YVsmTt74=
--- a/server/api/build.go
+++ b/server/api/build.go
@ -453,16 +453,36 @@ func PostBuild(c *gin.Context) {
 		}
 	}

-	// fetch the pipeline config from database
+	var pipelineFiles []*remote.FileMeta
+
+	// fetch the old pipeline config from database
 	configs, err := _store.ConfigsForBuild(build.ID)
 	if err != nil {
 		log.Error().Msgf("failure to get build config for %s. %s", repo.FullName, err)
 		_ = c.AbortWithError(404, err)
 		return
 	}
-	var yamls []*remote.FileMeta
+
 	for _, y := range configs {
-		yamls = append(yamls, &remote.FileMeta{Data: y.Data, Name: y.Name})
+		pipelineFiles = append(pipelineFiles, &remote.FileMeta{Data: y.Data, Name: y.Name})
+	}
+
+	// If config extension is active we should refetch the config in case something changed
+	if server.Config.Services.ConfigService.IsConfigured() {
+		currentFileMeta := make([]*remote.FileMeta, len(configs))
+		for i, cfg := range configs {
+			currentFileMeta[i] = &remote.FileMeta{Name: cfg.Name, Data: cfg.Data}
+		}
+
+		newConfig, useOld, err := server.Config.Services.ConfigService.FetchExternalConfig(c, repo, build, currentFileMeta)
+		if err != nil {
+			msg := fmt.Sprintf("On fetching external build config: %s", err)
+			c.String(http.StatusBadRequest, msg)
+			return
+		}
+		if !useOld {
+			pipelineFiles = newConfig
+		}
 	}

 	build.ID = 0
@ -515,7 +535,7 @@ func PostBuild(c *gin.Context) {
 		}
 	}

-	build, buildItems, err := createBuildItems(c, _store, build, user, repo, yamls, envs)
+	build, buildItems, err := createBuildItems(c, _store, build, user, repo, pipelineFiles, envs)
 	if err != nil {
 		msg := fmt.Sprintf("failure to createBuildItems for %s", repo.FullName)
 		log.Error().Err(err).Msg(msg)
--- a/server/api/hook.go
+++ b/server/api/hook.go
@ -177,7 +177,7 @@ func PostHook(c *gin.Context) {
 	}

 	// fetch the build file from the remote
-	configFetcher := shared.NewConfigFetcher(server.Config.Services.Remote, repoUser, repo, build)
+	configFetcher := shared.NewConfigFetcher(server.Config.Services.Remote, server.Config.Services.ConfigService, repoUser, repo, build)
 	remoteYamlConfigs, err := configFetcher.Fetch(c)
 	if err != nil {
 		msg := fmt.Sprintf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, build.Ref, repoUser.Login)
--- a/server/config.go
+++ b/server/config.go
@ -22,6 +22,7 @@ import (

 	"github.com/woodpecker-ci/woodpecker/server/logging"
 	"github.com/woodpecker-ci/woodpecker/server/model"
+	"github.com/woodpecker-ci/woodpecker/server/plugins/configuration"
 	"github.com/woodpecker-ci/woodpecker/server/pubsub"
 	"github.com/woodpecker-ci/woodpecker/server/queue"
 	"github.com/woodpecker-ci/woodpecker/server/remote"
@ -29,14 +30,15 @@ import (

 var Config = struct {
 	Services struct {
-		Pubsub     pubsub.Publisher
-		Queue      queue.Queue
-		Logs       logging.Log
-		Senders    model.SenderService
-		Secrets    model.SecretService
-		Registries model.RegistryService
-		Environ    model.EnvironService
-		Remote     remote.Remote
+		Pubsub        pubsub.Publisher
+		Queue         queue.Queue
+		Logs          logging.Log
+		Senders       model.SenderService
+		Secrets       model.SecretService
+		Registries    model.RegistryService
+		Environ       model.EnvironService
+		Remote        remote.Remote
+		ConfigService configuration.ConfigService
 	}
 	Storage struct {
 		// Users  model.UserStore
--- a/server/plugins/configuration/configuration.go
+++ b/server/plugins/configuration/configuration.go
@ -0,0 +1,127 @@
+package configuration
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/99designs/httpsignatures-go"
+
+	"github.com/woodpecker-ci/woodpecker/server/model"
+	"github.com/woodpecker-ci/woodpecker/server/remote"
+)
+
+type ConfigService struct {
+	endpoint string
+	secret   string
+}
+
+// Same as remote.FileMeta but with json tags and string data
+type config struct {
+	Name string `json:"name"`
+	Data string `json:"data"`
+}
+
+type requestStructure struct {
+	Repo          *model.Repo  `json:"repo"`
+	Build         *model.Build `json:"build"`
+	Configuration []*config    `json:"configs"`
+}
+
+type responseStructure struct {
+	Configs []config `json:"configs"`
+}
+
+func NewAPI(endpoint, secret string) ConfigService {
+	return ConfigService{endpoint: endpoint, secret: secret}
+}
+
+func (cp *ConfigService) IsConfigured() bool {
+	return cp.endpoint != ""
+}
+
+func (cp *ConfigService) FetchExternalConfig(ctx context.Context, repo *model.Repo, build *model.Build, currentFileMeta []*remote.FileMeta) (configData []*remote.FileMeta, useOld bool, err error) {
+	currentConfigs := make([]*config, len(currentFileMeta))
+	for i, pipe := range currentFileMeta {
+		currentConfigs[i] = &config{Name: pipe.Name, Data: string(pipe.Data)}
+	}
+
+	response, status, err := sendRequest(ctx, "POST", cp.endpoint, cp.secret, requestStructure{Repo: repo, Build: build, Configuration: currentConfigs})
+	if err != nil {
+		return nil, false, fmt.Errorf("Failed to fetch config via http (%d) %w", status, err)
+	}
+
+	var newFileMeta []*remote.FileMeta
+	if response != nil {
+		newFileMeta = make([]*remote.FileMeta, len(response.Configs))
+		for i, pipe := range response.Configs {
+			newFileMeta[i] = &remote.FileMeta{Name: pipe.Name, Data: []byte(pipe.Data)}
+		}
+	} else {
+		newFileMeta = make([]*remote.FileMeta, 0)
+	}
+
+	return newFileMeta, status == 204, nil
+}
+
+func sendRequest(ctx context.Context, method, path, signkey string, in interface{}) (response *responseStructure, statuscode int, err error) {
+	uri, err := url.Parse(path)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	// if we are posting or putting data, we need to
+	// write it to the body of the request.
+	var buf io.ReadWriter
+	if in != nil {
+		buf = new(bytes.Buffer)
+		jsonerr := json.NewEncoder(buf).Encode(in)
+		if jsonerr != nil {
+			return nil, 0, jsonerr
+		}
+	}
+
+	// creates a new http request to bitbucket.
+	req, err := http.NewRequestWithContext(ctx, method, uri.String(), buf)
+	if err != nil {
+		return nil, 0, err
+	}
+	if in != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	// Sign using the 'Signature' header
+	err = httpsignatures.DefaultSha256Signer.SignRequest("hmac-key", signkey, req)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 && resp.StatusCode != 204 {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, resp.StatusCode, err
+		}
+
+		return nil, resp.StatusCode, fmt.Errorf("Response: %s", string(body))
+	}
+
+	if resp.StatusCode == 204 {
+		return nil, resp.StatusCode, nil
+	}
+
+	// if no other errors parse and return the json response.
+	decodedResponse := new(responseStructure)
+	err = json.NewDecoder(resp.Body).Decode(decodedResponse)
+	return decodedResponse, resp.StatusCode, err
+}
--- a/server/shared/configFetcher.go
+++ b/server/shared/configFetcher.go
@ -8,6 +8,7 @@ import (
 	"time"

 	"github.com/rs/zerolog/log"
+	"github.com/woodpecker-ci/woodpecker/server/plugins/configuration"

 	"github.com/woodpecker-ci/woodpecker/server/model"
 	"github.com/woodpecker-ci/woodpecker/server/remote"
@ -18,37 +19,56 @@ type ConfigFetcher interface {
 }

 type configFetcher struct {
-	remote remote.Remote
-	user   *model.User
-	repo   *model.Repo
-	build  *model.Build
+	remote        remote.Remote
+	user          *model.User
+	repo          *model.Repo
+	build         *model.Build
+	configService configuration.ConfigService
 }

-func NewConfigFetcher(remote remote.Remote, user *model.User, repo *model.Repo, build *model.Build) ConfigFetcher {
+func NewConfigFetcher(remote remote.Remote, configurationService configuration.ConfigService, user *model.User, repo *model.Repo, build *model.Build) ConfigFetcher {
 	return &configFetcher{
-		remote: remote,
-		user:   user,
-		repo:   repo,
-		build:  build,
+		remote:        remote,
+		user:          user,
+		repo:          repo,
+		build:         build,
+		configService: configurationService,
 	}
 }

 // configFetchTimeout determine seconds the configFetcher wait until cancel fetch process
-var configFetchTimeout = 3 // seconds
+var configFetchTimeout = time.Second * 3

 // Fetch pipeline config from source forge
 func (cf *configFetcher) Fetch(ctx context.Context) (files []*remote.FileMeta, err error) {
 	log.Trace().Msgf("Start Fetching config for '%s'", cf.repo.FullName)

-	// try to fetch 3 times, timeout is one second longer each time
+	// try to fetch 3 times
 	for i := 0; i < 3; i++ {
-		files, err = cf.fetch(ctx, time.Second*time.Duration(configFetchTimeout), strings.TrimSpace(cf.repo.Config))
+		files, err = cf.fetch(ctx, configFetchTimeout, strings.TrimSpace(cf.repo.Config))
 		if err != nil {
 			log.Trace().Err(err).Msgf("%d. try failed", i+1)
 		}
 		if errors.Is(err, context.DeadlineExceeded) {
 			continue
 		}
+
+		if cf.configService.IsConfigured() {
+			fetchCtx, cancel := context.WithTimeout(ctx, configFetchTimeout)
+			defer cancel() // ok here as we only try http fetching once, returning on fail and success
+
+			log.Trace().Msgf("ConfigFetch[%s]: getting config from external http service", cf.repo.FullName)
+			newConfigs, useOld, err := cf.configService.FetchExternalConfig(fetchCtx, cf.repo, cf.build, files)
+			if err != nil {
+				log.Error().Msg("Got errror " + err.Error())
+				return nil, fmt.Errorf("On Fetching config via http : %s", err)
+			}
+
+			if !useOld {
+				return newConfigs, nil
+			}
+		}
+
 		return
 	}
 	return
--- a/server/shared/configFetcher_test.go
+++ b/server/shared/configFetcher_test.go
@ -2,14 +2,21 @@ package shared_test

 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
 	"path/filepath"
 	"testing"

+	"github.com/99designs/httpsignatures-go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"

 	"github.com/woodpecker-ci/woodpecker/server/model"
+	"github.com/woodpecker-ci/woodpecker/server/plugins/configuration"
 	"github.com/woodpecker-ci/woodpecker/server/remote"
 	"github.com/woodpecker-ci/woodpecker/server/remote/mocks"
 	"github.com/woodpecker-ci/woodpecker/server/shared"
@ -51,7 +58,7 @@ func TestFetch(t *testing.T) {
 			expectedError: false,
 		},
 		{
-			name:       "Default config - .woodpecker.yml",
+			name:       "Override via API with custom config",
 			repoConfig: "",
 			files: []file{{
 				name: ".woodpecker.yml",
@ -63,7 +70,7 @@ func TestFetch(t *testing.T) {
 			expectedError: false,
 		},
 		{
-			name:       "Default config - .drone.yml",
+			name:       "Use old config on 204 response",
 			repoConfig: "",
 			files: []file{{
 				name: ".drone.yml",
@ -237,6 +244,201 @@ func TestFetch(t *testing.T) {

 			configFetcher := shared.NewConfigFetcher(
 				r,
+				configuration.NewAPI("", ""),
+				&model.User{Token: "xxx"},
+				repo,
+				&model.Build{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"},
+			)
+			files, err := configFetcher.Fetch(context.Background())
+			if tt.expectedError && err == nil {
+				t.Fatal("expected an error")
+			} else if !tt.expectedError && err != nil {
+				t.Fatal("error fetching config:", err)
+			}
+
+			matchingFiles := make([]string, len(files))
+			for i := range files {
+				matchingFiles[i] = files[i].Name
+			}
+			assert.ElementsMatch(t, tt.expectedFileNames, matchingFiles, "expected some other pipeline files")
+		})
+	}
+}
+
+func TestFetchFromConfigService(t *testing.T) {
+	t.Parallel()
+
+	type file struct {
+		name string
+		data []byte
+	}
+
+	dummyData := []byte("TEST")
+
+	testTable := []struct {
+		name              string
+		repoConfig        string
+		files             []file
+		expectedFileNames []string
+		expectedError     bool
+	}{
+		{
+			name:              "External Fetch empty repo",
+			repoConfig:        "",
+			files:             []file{},
+			expectedFileNames: []string{"override1", "override2", "override3"},
+			expectedError:     false,
+		},
+		{
+			name:       "Default config - Additional sub-folders",
+			repoConfig: "",
+			files: []file{{
+				name: ".woodpecker/test.yml",
+				data: dummyData,
+			}, {
+				name: ".woodpecker/sub-folder/config.yml",
+				data: dummyData,
+			}},
+			expectedFileNames: []string{"override1", "override2", "override3"},
+			expectedError:     false,
+		},
+		{
+			name:       "Fetch empty",
+			repoConfig: " ",
+			files: []file{{
+				name: ".woodpecker/.keep",
+				data: dummyData,
+			}, {
+				name: ".woodpecker.yml",
+				data: nil,
+			}, {
+				name: ".drone.yml",
+				data: dummyData,
+			}},
+			expectedFileNames: []string{},
+			expectedError:     true,
+		},
+		{
+			name:       "Use old config",
+			repoConfig: ".my-ci-folder/",
+			files: []file{{
+				name: ".woodpecker/test.yml",
+				data: dummyData,
+			}, {
+				name: ".woodpecker.yml",
+				data: dummyData,
+			}, {
+				name: ".drone.yml",
+				data: dummyData,
+			}, {
+				name: ".my-ci-folder/test.yml",
+				data: dummyData,
+			}},
+			expectedFileNames: []string{
+				".my-ci-folder/test.yml",
+			},
+			expectedError: false,
+		},
+	}
+
+	httpSigSecret := "wykf9frJbGXwSHcJ7AQF4tlfXUo0Tkixh57WPEXMyWVgkxIsAarYa2Hb8UTwPpbqO0N3NueKwjv4DVhPgvQjGur3LuCbiGHbBoaL1X5gZ9oyxD2lBHndoNxifDyNH7tNPw3Lh5lX2MSrWP1yuqHp8Sgm7fX8pLTjaKKFgFIKlODd"
+
+	fixtureHandler := func(w http.ResponseWriter, r *http.Request) {
+		// check signature
+		signature, err := httpsignatures.FromRequest(r)
+		if err != nil {
+			http.Error(w, "Invalid or Missing Signature", http.StatusBadRequest)
+			return
+		}
+		if !signature.IsValid(httpSigSecret, r) {
+			http.Error(w, "Invalid Signature", http.StatusBadRequest)
+			return
+		}
+
+		type config struct {
+			Name string `json:"name"`
+			Data string `json:"data"`
+		}
+
+		type incoming struct {
+			Repo          *model.Repo  `json:"repo"`
+			Build         *model.Build `json:"build"`
+			Configuration []*config    `json:"config"`
+		}
+
+		var req incoming
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			log.Printf("Error reading body: %v", err)
+			http.Error(w, "can't read body", http.StatusBadRequest)
+			return
+		}
+		err = json.Unmarshal(body, &req)
+		if err != nil {
+			http.Error(w, "Failed to parse JSON"+err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		if req.Repo.Name == "Fetch empty" {
+			w.WriteHeader(404)
+			return
+		}
+
+		if req.Repo.Name == "Use old config" {
+			w.WriteHeader(204)
+			return
+		}
+
+		fmt.Fprint(w, `{
+			"configs": [
+					{
+							"name": "override1",
+							"data": "some new pipelineconfig \n pipe, pipe, pipe"
+					},
+					{
+							"name": "override2",
+							"data": "some new pipelineconfig \n pipe, pipe, pipe"
+					},
+					{
+							"name": "override3",
+							"data": "some new pipelineconfig \n pipe, pipe, pipe"
+					}
+			]
+}`)
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(fixtureHandler))
+	defer ts.Close()
+	configAPI := configuration.NewAPI(ts.URL, httpSigSecret)
+
+	for _, tt := range testTable {
+		t.Run(tt.name, func(t *testing.T) {
+			repo := &model.Repo{Owner: "laszlocph", Name: tt.name, Config: tt.repoConfig} // Using test name as repo name to provide different responses in mock server
+
+			r := new(mocks.Remote)
+			dirs := map[string][]*remote.FileMeta{}
+			for _, file := range tt.files {
+				r.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil)
+				path := filepath.Dir(file.name)
+				if path != "." {
+					dirs[path] = append(dirs[path], &remote.FileMeta{
+						Name: file.name,
+						Data: file.data,
+					})
+				}
+			}
+
+			for path, files := range dirs {
+				r.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, path).Return(files, nil)
+			}
+
+			// if the previous mocks do not match return not found errors
+			r.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("File not found"))
+			r.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("Directory not found"))
+
+			configFetcher := shared.NewConfigFetcher(
+				r,
+				configAPI,
 				&model.User{Token: "xxx"},
 				repo,
 				&model.Build{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"},
--- a/vendor/github.com/99designs/httpsignatures-go/.travis.yml
+++ b/vendor/github.com/99designs/httpsignatures-go/.travis.yml
@ -0,0 +1,13 @@
+language: go
+
+go:
+  - 1.2
+  - 1.3
+  - 1.4
+  - 1.5
+  - 1.6
+  - 1.7
+  - 1.8
+
+install:
+  - go get github.com/stretchr/testify/assert
--- a/vendor/github.com/99designs/httpsignatures-go/LICENSE
+++ b/vendor/github.com/99designs/httpsignatures-go/LICENSE
@ -0,0 +1,22 @@
+
+The MIT License (MIT)
+
+Copyright (c) 2016 99designs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/vendor/github.com/99designs/httpsignatures-go/README.md
+++ b/vendor/github.com/99designs/httpsignatures-go/README.md
@ -0,0 +1,9 @@
+httpsignatures-go
+=================
+[![GoDoc](https://godoc.org/github.com/99designs/httpsignatures-go?status.svg)](https://godoc.org/github.com/99designs/httpsignatures-go)
+[![Build Status](https://travis-ci.org/99designs/httpsignatures-go.svg)](https://travis-ci.org/99designs/httpsignatures-go)
+
+
+Golang library for the [http-signatures spec](https://tools.ietf.org/html/draft-cavage-http-signatures).
+
+See https://godoc.org/github.com/99designs/httpsignatures-go for documentation and examples
--- a/vendor/github.com/99designs/httpsignatures-go/algorithm.go
+++ b/vendor/github.com/99designs/httpsignatures-go/algorithm.go
@ -0,0 +1,31 @@
+package httpsignatures
+
+import (
+	"crypto/sha1"
+	"crypto/sha256"
+	"errors"
+	"hash"
+)
+
+var (
+	AlgorithmHmacSha256 = &Algorithm{"hmac-sha256", sha256.New}
+	AlgorithmHmacSha1   = &Algorithm{"hmac-sha1", sha1.New}
+
+	ErrorUnknownAlgorithm = errors.New("Unknown Algorithm")
+)
+
+type Algorithm struct {
+	name string
+	hash func() hash.Hash
+}
+
+func algorithmFromString(name string) (*Algorithm, error) {
+	switch name {
+	case AlgorithmHmacSha1.name:
+		return AlgorithmHmacSha1, nil
+	case AlgorithmHmacSha256.name:
+		return AlgorithmHmacSha256, nil
+	}
+
+	return nil, ErrorUnknownAlgorithm
+}
--- a/vendor/github.com/99designs/httpsignatures-go/signature.go
+++ b/vendor/github.com/99designs/httpsignatures-go/signature.go
@ -0,0 +1,202 @@
+// httpsignatures is a golang implementation of the http-signatures spec
+// found at https://tools.ietf.org/html/draft-cavage-http-signatures
+package httpsignatures
+
+import (
+	"crypto/hmac"
+	"crypto/subtle"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+)
+
+const (
+	headerSignature     = "Signature"
+	headerAuthorization = "Authorization"
+
+	RequestTarget = "(request-target)"
+
+	authScheme = "Signature "
+)
+
+var (
+	ErrorNoSignatureHeader = errors.New("No Signature header found in request")
+
+	signatureRegex = regexp.MustCompile(`(\w+)="([^"]*)"`)
+)
+
+// Signature is the hashed key + headers, either from a request or a signer
+type Signature struct {
+	KeyID     string
+	Algorithm *Algorithm
+	Headers   HeaderList
+	Signature string
+}
+
+// FromRequest creates a new Signature from the Request
+// both Signature and Authorization http headers are supported.
+func FromRequest(r *http.Request) (*Signature, error) {
+	if s, ok := r.Header[headerSignature]; ok {
+		return FromString(s[0])
+	}
+	if a, ok := r.Header[headerAuthorization]; ok {
+		return FromString(strings.TrimPrefix(a[0], authScheme))
+	}
+	return nil, ErrorNoSignatureHeader
+}
+
+// FromString creates a new Signature from its encoded form,
+// eg `keyId="a",algorithm="b",headers="c",signature="d"`
+func FromString(in string) (*Signature, error) {
+	var res Signature = Signature{}
+	var key string
+	var value string
+
+	for _, m := range signatureRegex.FindAllStringSubmatch(in, -1) {
+		key = m[1]
+		value = m[2]
+
+		if key == "keyId" {
+			res.KeyID = value
+		} else if key == "algorithm" {
+			alg, err := algorithmFromString(value)
+			if err != nil {
+				return nil, err
+			}
+			res.Algorithm = alg
+		} else if key == "headers" {
+			res.Headers = headerListFromString(value)
+		} else if key == "signature" {
+			res.Signature = value
+		} else {
+			return nil, errors.New(fmt.Sprintf("Unexpected key in signature '%s'", key))
+		}
+	}
+
+	if len(res.Signature) == 0 {
+		return nil, errors.New("Missing signature")
+	}
+
+	if len(res.KeyID) == 0 {
+		return nil, errors.New("Missing keyId")
+	}
+
+	if res.Algorithm == nil {
+		return nil, errors.New("Missing algorithm")
+	}
+
+	return &res, nil
+}
+
+// String returns the encoded form of the Signature
+func (s Signature) String() string {
+	str := fmt.Sprintf(
+		`keyId="%s",algorithm="%s",signature="%s"`,
+		s.KeyID,
+		s.Algorithm.name,
+		s.Signature,
+	)
+
+	if len(s.Headers) > 0 {
+		str += fmt.Sprintf(`,headers="%s"`, s.Headers.String())
+	}
+
+	return str
+}
+
+func (s Signature) calculateSignature(key string, r *http.Request) (string, error) {
+	hash := hmac.New(s.Algorithm.hash, []byte(key))
+
+	signingString, err := s.Headers.signingString(r)
+	if err != nil {
+		return "", err
+	}
+
+	hash.Write([]byte(signingString))
+
+	return base64.StdEncoding.EncodeToString(hash.Sum(nil)), nil
+}
+
+// Sign this signature using the given key
+func (s *Signature) sign(key string, r *http.Request) error {
+	sig, err := s.calculateSignature(key, r)
+	if err != nil {
+		return err
+	}
+
+	s.Signature = sig
+	return nil
+}
+
+// IsValid validates this signature for the given key
+func (s Signature) IsValid(key string, r *http.Request) bool {
+	if !s.Headers.hasDate() {
+		return false
+	}
+
+	sig, err := s.calculateSignature(key, r)
+	if err != nil {
+		return false
+	}
+
+	return subtle.ConstantTimeCompare([]byte(s.Signature), []byte(sig)) == 1
+}
+
+type HeaderList []string
+
+func headerListFromString(list string) HeaderList {
+	return strings.Split(strings.ToLower(string(list)), " ")
+}
+
+func (h HeaderList) String() string {
+	return strings.ToLower(strings.Join(h, " "))
+}
+
+func (h HeaderList) hasDate() bool {
+	for _, header := range h {
+		if header == "date" {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (h HeaderList) signingString(req *http.Request) (string, error) {
+	lines := []string{}
+
+	for _, header := range h {
+		if header == RequestTarget {
+			lines = append(lines, requestTargetLine(req))
+		} else {
+			line, err := headerLine(req, header)
+			if err != nil {
+				return "", err
+			}
+			lines = append(lines, line)
+		}
+	}
+
+	return strings.Join(lines, "\n"), nil
+}
+
+func requestTargetLine(req *http.Request) string {
+	var url string = ""
+	if req.URL != nil {
+		url = req.URL.RequestURI()
+	}
+
+	return fmt.Sprintf("%s: %s %s", RequestTarget, strings.ToLower(req.Method), url)
+}
+
+func headerLine(req *http.Request, header string) (string, error) {
+
+	if value := req.Header.Get(header); value != "" {
+		return fmt.Sprintf("%s: %s", header, value), nil
+	}
+
+	return "", errors.New(fmt.Sprintf("Missing required header '%s'", header))
+}
--- a/vendor/github.com/99designs/httpsignatures-go/signer.go
+++ b/vendor/github.com/99designs/httpsignatures-go/signer.go
@ -0,0 +1,79 @@
+package httpsignatures
+
+import (
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Signer is used to create a signature for a given request.
+type Signer struct {
+	algorithm *Algorithm
+	headers   HeaderList
+}
+
+var (
+	// DefaultSha1Signer will sign requests with the url and date using the SHA1 algorithm.
+	// Users are encouraged to create their own signer with the headers they require.
+	DefaultSha1Signer = NewSigner(AlgorithmHmacSha1, RequestTarget, "date")
+
+	// DefaultSha256Signer will sign requests with the url and date using the SHA256 algorithm.
+	// Users are encouraged to create their own signer with the headers they require.
+	DefaultSha256Signer = NewSigner(AlgorithmHmacSha256, RequestTarget, "date")
+)
+
+func NewSigner(algorithm *Algorithm, headers ...string) *Signer {
+	hl := HeaderList{}
+
+	for _, header := range headers {
+		hl = append(hl, strings.ToLower(header))
+	}
+
+	return &Signer{
+		algorithm: algorithm,
+		headers:   hl,
+	}
+}
+
+// SignRequest adds a http signature to the Signature: HTTP Header
+func (s Signer) SignRequest(id, key string, r *http.Request) error {
+	sig, err := s.buildSignature(id, key, r)
+	if err != nil {
+		return err
+	}
+
+	r.Header.Add(headerSignature, sig.String())
+
+	return nil
+}
+
+// AuthRequest adds a http signature to the Authorization: HTTP Header
+func (s Signer) AuthRequest(id, key string, r *http.Request) error {
+	sig, err := s.buildSignature(id, key, r)
+	if err != nil {
+		return err
+	}
+
+	r.Header.Add(headerAuthorization, authScheme+sig.String())
+
+	return nil
+}
+
+func (s Signer) buildSignature(id, key string, r *http.Request) (*Signature, error) {
+	if r.Header.Get("date") == "" {
+		r.Header.Set("date", time.Now().Format(time.RFC1123))
+	}
+
+	sig := &Signature{
+		KeyID:     id,
+		Algorithm: s.algorithm,
+		Headers:   s.headers,
+	}
+
+	err := sig.sign(key, r)
+	if err != nil {
+		return nil, err
+	}
+
+	return sig, nil
+}
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@ -3,6 +3,9 @@
 # code.gitea.io/sdk/gitea v0.15.0
 ## explicit
 code.gitea.io/sdk/gitea
+# github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e
+## explicit
+github.com/99designs/httpsignatures-go
 # github.com/Antonboom/errname v0.1.5
 github.com/Antonboom/errname/pkg/analyzer
 # github.com/Antonboom/nilnil v0.1.0