yume/act
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

matrix is done

Browse source 
Signed-off-by: Casey Lee <cplee@nektos.com>
This commit is contained in:
Casey Lee 2020-02-17 10:11:16 -08:00
parent 5b7019cd0b
commit f8fb88816a
No known key found for this signature in database
GPG key ID: 1899120ECD0A1784
8 changed files with 219 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/root.go
View file

@ -62,7 +62,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str
var eventName string
if len(args) > 0 {
eventName = args[0]
} else if events := planner.GetEvents(); len(events) > 1 {
} else if events := planner.GetEvents(); len(events) > 0 {
// set default event type to first event
// this way user dont have to specify the event.
log.Debugf("Using detected workflow event: %s", events[0])

54
pkg/common/cartesian.go Normal file
View file

@ -0,0 +1,54 @@
package common
// CartesianProduct takes map of lists and returns list of unique tuples
func CartesianProduct(mapOfLists map[string][]interface{}) []map[string]interface{} {
listNames := make([]string, 0)
lists := make([][]interface{}, 0)
for k, v := range mapOfLists {
listNames = append(listNames, k)
lists = append(lists, v)
}
listCart := cartN(lists...)
rtn := make([]map[string]interface{}, 0)
for _, list := range listCart {
vMap := make(map[string]interface{})
for i, v := range list {
vMap[listNames[i]] = v
}
rtn = append(rtn, vMap)
}
return rtn
}
func cartN(a ...[]interface{}) [][]interface{} {
c := 1
for _, a := range a {
c *= len(a)
}
if c == 0 {
return nil
}
p := make([][]interface{}, c)
b := make([]interface{}, c*len(a))
n := make([]int, len(a))
s := 0
for i := range p {
e := s + len(a)
pi := b[s:e]
p[i] = pi
s = e
for j, n := range n {
pi[j] = a[j][n]
}
for j := len(n) - 1; j >= 0; j-- {
n[j]++
if n[j] < len(a[j]) {
break
}
n[j] = 0
}
}
return p
}

28
pkg/common/cartesian_test.go Normal file
View file

@ -0,0 +1,28 @@
package common
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestCartisianProduct(t *testing.T) {
assert := assert.New(t)
input := map[string][]interface{}{
"foo": []interface{}{1, 2, 3, 4},
"bar": []interface{}{"a", "b", "c"},
"baz": []interface{}{false, true},
}
output := CartesianProduct(input)
assert.Len(output, 24)
for _, v := range output {
assert.Len(v, 3)
assert.Contains(v, "foo")
assert.Contains(v, "bar")
assert.Contains(v, "baz")
}
}

3
pkg/model/workflow.go
View file

@ -187,6 +187,9 @@ func ReadWorkflow(in io.Reader) (*Workflow, error) {
func (w *Workflow) GetJob(jobID string) *Job {
for id, j := range w.Jobs {
if jobID == id {
if j.Name == "" {
j.Name = id
}
return j
}
}

65
pkg/runner/run_context.go
View file

@ -24,16 +24,16 @@ import (
// RunContext contains info about current job
type RunContext struct {
Config *Config
Matrix map[string]interface{}
Run *model.Run
EventJSON string
Env map[string]string
Tempdir string
ExtraPath []string
CurrentStep string
StepResults map[string]*stepResult
PlatformName string
Config *Config
Matrix map[string]interface{}
Run *model.Run
EventJSON string
Env map[string]string
Tempdir string
ExtraPath []string
CurrentStep string
StepResults map[string]*stepResult
ExprEval ExpressionEvaluator
}
type stepResult struct {
@ -56,9 +56,6 @@ func (rc *RunContext) Close(ctx context.Context) error {
// Executor returns a pipeline executor for all the steps in the job
func (rc *RunContext) Executor() common.Executor {
if img := platformImage(rc.PlatformName); img == "" {
return common.NewInfoExecutor(" \U0001F6A7 Skipping unsupported platform '%s'", rc.PlatformName)
}
err := rc.setupTempDir()
if err != nil {
@ -77,6 +74,13 @@ func (rc *RunContext) Executor() common.Executor {
Success: true,
Outputs: make(map[string]string),
}
rc.ExprEval = rc.NewStepExpressionEvaluator(s)
if !rc.EvalBool(s.If) {
log.Debugf("Skipping step '%s' due to '%s'", s.String(), s.If)
return nil
}
common.Logger(ctx).Infof("\u2B50 Run %s", s)
err := rc.newStepExecutor(s)(ctx)
if err == nil {
@ -88,7 +92,36 @@ func (rc *RunContext) Executor() common.Executor {
return err
})
}
return common.NewPipelineExecutor(steps...).Finally(rc.Close)
return func(ctx context.Context) error {
defer rc.Close(ctx)
job := rc.Run.Job()
log := common.Logger(ctx)
if !rc.EvalBool(job.If) {
log.Debugf("Skipping job '%s' due to '%s'", job.Name, job.If)
return nil
}
platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn)
if img := platformImage(platformName); img == "" {
log.Infof(" \U0001F6A7 Skipping unsupported platform '%s'", platformName)
return nil
}
return common.NewPipelineExecutor(steps...)(ctx)
}
}
// EvalBool evaluates an expression against current run context
func (rc *RunContext) EvalBool(expr string) bool {
if expr != "" {
v, err := rc.ExprEval.Evaluate(expr)
if err != nil {
log.Errorf("Error evaluating expression '%s' - %v", expr, err)
return false
}
return v == "true"
}
return true
}
func mergeMaps(maps ...map[string]string) map[string]string {
@ -141,10 +174,10 @@ func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Ex
}
var cmd, entrypoint []string
if containerSpec.Args != "" {
cmd = strings.Fields(containerSpec.Args)
cmd = strings.Fields(rc.ExprEval.Interpolate(containerSpec.Args))
}
if containerSpec.Entrypoint != "" {
entrypoint = strings.Fields(containerSpec.Entrypoint)
entrypoint = strings.Fields(rc.ExprEval.Interpolate(containerSpec.Entrypoint))
}
rawLogger := common.Logger(ctx).WithField("raw_output", true)

65
pkg/runner/runner.go
View file

@ -53,18 +53,49 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
for _, stage := range plan.Stages {
stageExecutor := make([]common.Executor, 0)
for _, run := range stage.Runs {
// TODO - don't just grab first index of each dimension
matrix := make(map[string]interface{})
if run.Job().Strategy != nil {
for mkey, mvals := range run.Job().Strategy.Matrix {
if mkey == "include" || mkey == "exclude" {
continue
}
matrix[mkey] = mvals[0]
job := run.Job()
matrixes := make([]map[string]interface{}, 0)
if job.Strategy != nil {
includes := make([]map[string]interface{}, 0)
for _, v := range job.Strategy.Matrix["include"] {
includes = append(includes, v.(map[string]interface{}))
}
delete(job.Strategy.Matrix, "include")
excludes := make([]map[string]interface{}, 0)
for _, v := range job.Strategy.Matrix["exclude"] {
excludes = append(excludes, v.(map[string]interface{}))
}
delete(job.Strategy.Matrix, "exclude")
matrixProduct := common.CartesianProduct(job.Strategy.Matrix)
MATRIX:
for _, matrix := range matrixProduct {
for _, exclude := range excludes {
if commonKeysMatch(matrix, exclude) {
log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude)
continue MATRIX
}
}
for _, include := range includes {
if commonKeysMatch(matrix, include) {
log.Debugf("Setting add'l values on matrix '%v' due to include '%v'", matrix, include)
for k, v := range include {
matrix[k] = v
}
}
}
matrixes = append(matrixes, matrix)
}
} else {
matrixes = append(matrixes, make(map[string]interface{}))
}
stageExecutor = append(stageExecutor, runner.NewRunExecutor(run, matrix))
for _, matrix := range matrixes {
stageExecutor = append(stageExecutor, runner.NewRunExecutor(run, matrix))
}
}
pipeline = append(pipeline, common.NewParallelExecutor(stageExecutor...))
}
@ -72,6 +103,15 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor {
return common.NewPipelineExecutor(pipeline...)
}
func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool {
for aKey, aVal := range a {
if bVal, ok := b[aKey]; ok && aVal != bVal {
return false
}
}
return true
}
func (runner *runnerImpl) NewRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor {
rc := new(RunContext)
rc.Config = runner.config
@ -79,11 +119,12 @@ func (runner *runnerImpl) NewRunExecutor(run *model.Run, matrix map[string]inter
rc.EventJSON = runner.eventJSON
rc.StepResults = make(map[string]*stepResult)
rc.Matrix = matrix
ee := rc.NewExpressionEvaluator()
rc.PlatformName = ee.Interpolate(run.Job().RunsOn)
rc.ExprEval = rc.NewExpressionEvaluator()
return func(ctx context.Context) error {
ctx = WithJobLogger(ctx, rc.Run.String())
if len(rc.Matrix) > 0 {
common.Logger(ctx).Infof("\U0001F9EA Matrix: %v", rc.Matrix)
}
return rc.Executor()(ctx)
}
}

46
pkg/runner/step.go
View file

@ -15,17 +15,33 @@ import (
log "github.com/sirupsen/logrus"
)
func (rc *RunContext) StepEnv(step *model.Step) map[string]string {
var env map[string]string
job := rc.Run.Job()
if job.Container != nil {
env = mergeMaps(rc.GetEnv(), job.Container.Env, step.GetEnv())
} else {
env = mergeMaps(rc.GetEnv(), step.GetEnv())
}
for k, v := range env {
env[k] = rc.ExprEval.Interpolate(v)
}
return env
}
func (rc *RunContext) setupEnv(containerSpec *model.ContainerSpec, step *model.Step) common.Executor {
return func(ctx context.Context) error {
containerSpec.Env = rc.withGithubEnv(rc.StepEnv(step))
return nil
}
}
func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor {
ee := rc.NewStepExpressionEvaluator(step)
job := rc.Run.Job()
containerSpec := new(model.ContainerSpec)
containerSpec.Env = rc.withGithubEnv(rc.StepEnv(step))
containerSpec.Name = rc.createContainerName(step.ID)
for k, v := range containerSpec.Env {
containerSpec.Env[k] = ee.Interpolate(v)
}
switch step.Type() {
case model.StepTypeRun:
if job.Container != nil {
@ -34,9 +50,11 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor {
containerSpec.Volumes = job.Container.Volumes
containerSpec.Options = job.Container.Options
} else {
containerSpec.Image = platformImage(rc.PlatformName)
platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn)
containerSpec.Image = platformImage(platformName)
}
return common.NewPipelineExecutor(
rc.setupEnv(containerSpec, step),
rc.setupShellCommand(containerSpec, step.Shell, step.Run),
rc.pullImage(containerSpec),
rc.runContainer(containerSpec),
@ -47,6 +65,7 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor {
containerSpec.Entrypoint = step.With["entrypoint"]
containerSpec.Args = step.With["args"]
return common.NewPipelineExecutor(
rc.setupEnv(containerSpec, step),
rc.pullImage(containerSpec),
rc.runContainer(containerSpec),
)
@ -54,6 +73,7 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor {
case model.StepTypeUsesActionLocal:
containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest")
return common.NewPipelineExecutor(
rc.setupEnv(containerSpec, step),
rc.setupAction(containerSpec, filepath.Join(rc.Config.Workdir, step.Uses)),
applyWith(containerSpec, step),
rc.pullImage(containerSpec),
@ -78,6 +98,7 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor {
Ref: remoteAction.Ref,
Dir: cloneDir,
}),
rc.setupEnv(containerSpec, step),
rc.setupAction(containerSpec, filepath.Join(cloneDir, remoteAction.Path)),
applyWith(containerSpec, step),
rc.pullImage(containerSpec),
@ -100,15 +121,6 @@ func applyWith(containerSpec *model.ContainerSpec, step *model.Step) common.Exec
}
}
// StepEnv returns the env for a step
func (rc *RunContext) StepEnv(step *model.Step) map[string]string {
job := rc.Run.Job()
if job.Container != nil {
return mergeMaps(rc.GetEnv(), job.Container.Env, step.GetEnv())
}
return mergeMaps(rc.GetEnv(), step.GetEnv())
}
func (rc *RunContext) setupShellCommand(containerSpec *model.ContainerSpec, shell string, run string) common.Executor {
return func(ctx context.Context) error {
shellCommand := ""
@ -140,6 +152,8 @@ func (rc *RunContext) setupShellCommand(containerSpec *model.ContainerSpec, shel
return err
}
run = rc.ExprEval.Interpolate(run)
if _, err := tempScript.WriteString(run); err != nil {
return err
}

2
pkg/runner/testdata/matrix/push.yml vendored
View file

@ -5,7 +5,7 @@ jobs:
build:
runs-on: ${{ matrix.os }}
steps:
- run: echo ${NODE_VERSION} | grep 4
- run: echo ${NODE_VERSION} | grep ${{ matrix.node }}
env:
NODE_VERSION: ${{ matrix.node }}
strategy: