From b9a7bc620228df56dcc0684f09391ea481c9fb04 Mon Sep 17 00:00:00 2001 From: Mark DeLillo Date: Fri, 2 Apr 2021 16:40:44 -0400 Subject: [PATCH] Add support for composite actions (#514) * Add support for composite actions * Fix to make more complex composite actions work * Fix to make more complex composite actions work * Let's validate the steps in the composite steps to fail on uses and run's without shell, like the real world * Add support for composite actions * Add workflow to test composite actions * Log instead of panicing when output is mismatched * Merge maps so environment variables are not lost * Remove Debug * Correect merge error * Remove invalid composite tests. * Fix composite test Co-authored-by: Casey Lee Co-authored-by: monkers Co-authored-by: Mike Moncrieffe <69815687+mikemonkers@users.noreply.github.com> --- .github/actions/composite/action.yml | 22 +++++++ .github/actions/composite/script.sh | 1 + .github/workflows/test-composite.yml | 25 ++++++++ pkg/model/action.go | 7 ++- pkg/model/workflow.go | 9 +++ pkg/runner/command.go | 17 ++++- pkg/runner/run_context.go | 41 ++++++++---- pkg/runner/runner_test.go | 2 + pkg/runner/step_context.go | 62 +++++++++++++++++++ .../composite_action/action.yml | 48 ++++++++++++++ pkg/runner/testdata/uses-composite/push.yml | 18 ++++++ 11 files changed, 236 insertions(+), 16 deletions(-) create mode 100644 .github/actions/composite/action.yml create mode 100755 .github/actions/composite/script.sh create mode 100644 .github/workflows/test-composite.yml mode change 100644 => 100755 pkg/runner/command.go mode change 100644 => 100755 pkg/runner/run_context.go mode change 100644 => 100755 pkg/runner/runner_test.go mode change 100644 => 100755 pkg/runner/step_context.go create mode 100644 pkg/runner/testdata/uses-composite/composite_action/action.yml create mode 100755 pkg/runner/testdata/uses-composite/push.yml diff --git a/.github/actions/composite/action.yml b/.github/actions/composite/action.yml new file mode 100644 index 0000000..f8132bf --- /dev/null +++ b/.github/actions/composite/action.yml @@ -0,0 +1,22 @@ +name: 'Test Composite Action' +description: 'Test Composite Action' +inputs: + input: + description: 'String to echo' + required: true + default: 'Hello World' +outputs: + output: + description: 'Output string' + value: ${{ steps.set-output.outputs.string }} +runs: + using: "composite" + steps: + - run: echo Hello ${{ inputs.input }}. + shell: bash + - id: set-output + run: echo "::set-output name=string::output string" + shell: bash + - run: | + ${{ github.action_path }}/script.sh + shell: bash diff --git a/.github/actions/composite/script.sh b/.github/actions/composite/script.sh new file mode 100755 index 0000000..25a5b1c --- /dev/null +++ b/.github/actions/composite/script.sh @@ -0,0 +1 @@ +echo "Output from script" \ No newline at end of file diff --git a/.github/workflows/test-composite.yml b/.github/workflows/test-composite.yml new file mode 100644 index 0000000..2282afc --- /dev/null +++ b/.github/workflows/test-composite.yml @@ -0,0 +1,25 @@ +name: "Test composite actions" +on: push + +env: + ACT: true + +jobs: + test-composite-actions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: "✅ I should echo default 'Hello World'" + id: step0 + uses: ./.github/actions/composite + + - name: "Check action output" + if: steps.step0.outputs.output != 'output string' + run: echo "expected output to be 'output string' but it wasn't" + + - name: "✅ I should echo 'Test string'" + id: step1 + uses: ./.github/actions/composite + with: + input: 'Test string' \ No newline at end of file diff --git a/pkg/model/action.go b/pkg/model/action.go index 8a6c045..d350b51 100644 --- a/pkg/model/action.go +++ b/pkg/model/action.go @@ -20,10 +20,11 @@ func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(interface{}) error) error // Force input to lowercase for case insensitive comparison format := ActionRunsUsing(strings.ToLower(using)) switch format { - case ActionRunsUsingNode12, ActionRunsUsingDocker: + case ActionRunsUsingNode12, ActionRunsUsingDocker, ActionRunsUsingComposite: *a = format default: return fmt.Errorf(fmt.Sprintf("The runs.using key in action.yml must be one of: %v, got %s", []string{ + ActionRunsUsingComposite, ActionRunsUsingDocker, ActionRunsUsingNode12, }, format)) @@ -36,6 +37,8 @@ const ( ActionRunsUsingNode12 = "node12" // ActionRunsUsingDocker for running with docker ActionRunsUsingDocker = "docker" + // ActionRunsUsingComposite for running composite + ActionRunsUsingComposite = "composite" ) // ActionRuns are a field in Action @@ -46,6 +49,7 @@ type ActionRuns struct { Image string `yaml:"image"` Entrypoint []string `yaml:"entrypoint"` Args []string `yaml:"args"` + Steps []Step `yaml:"steps"` } // Action describes a metadata file for GitHub actions. The metadata filename must be either action.yml or action.yaml. The data in the metadata file defines the inputs, outputs and main entrypoint for your action. @@ -72,6 +76,7 @@ type Input struct { // Output parameters allow you to declare data that an action sets. Actions that run later in a workflow can use the output data set in previously run actions. For example, if you had an action that performed the addition of two inputs (x + y = z), the action could output the sum (z) for other actions to use as an input. type Output struct { Description string `yaml:"description"` + Value string `yaml:"value"` } // ReadAction reads an action from a reader diff --git a/pkg/model/workflow.go b/pkg/model/workflow.go index 255b5e0..8fe9354 100644 --- a/pkg/model/workflow.go +++ b/pkg/model/workflow.go @@ -312,6 +312,15 @@ func (s *Step) Type() StepType { return StepTypeUsesActionRemote } +func (s *Step) Validate() error { + if s.Type() != StepTypeRun { + return fmt.Errorf("(StepID: %s): Unexpected value 'uses'", s.String()) + } else if s.Shell == "" { + return fmt.Errorf("(StepID: %s): Required property is missing: 'shell'", s.String()) + } + return nil +} + // ReadWorkflow returns a list of jobs for a given workflow file reader func ReadWorkflow(in io.Reader) (*Workflow, error) { w := new(Workflow) diff --git a/pkg/runner/command.go b/pkg/runner/command.go old mode 100644 new mode 100755 index 0b230a5..b7ba0d3 --- a/pkg/runner/command.go +++ b/pkg/runner/command.go @@ -77,8 +77,21 @@ func (rc *RunContext) setEnv(ctx context.Context, kvPairs map[string]string, arg rc.Env[kvPairs["name"]] = arg } func (rc *RunContext) setOutput(ctx context.Context, kvPairs map[string]string, arg string) { - common.Logger(ctx).Infof(" \U00002699 ::set-output:: %s=%s", kvPairs["name"], arg) - rc.StepResults[rc.CurrentStep].Outputs[kvPairs["name"]] = arg + stepID := rc.CurrentStep + outputName := kvPairs["name"] + if outputMapping, ok := rc.OutputMappings[MappableOutput{StepID: stepID, OutputName: outputName}]; ok { + stepID = outputMapping.StepID + outputName = outputMapping.OutputName + } + + result, ok := rc.StepResults[stepID] + if !ok { + common.Logger(ctx).Infof(" \U00002757 no outputs used step '%s'", stepID) + return + } + + common.Logger(ctx).Infof(" \U00002699 ::set-output:: %s=%s", outputName, arg) + result.Outputs[outputName] = arg } func (rc *RunContext) addPath(ctx context.Context, arg string) { common.Logger(ctx).Infof(" \U00002699 ::add-path:: %s", arg) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go old mode 100644 new mode 100755 index ee9937e..eacc231 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -20,17 +20,23 @@ import ( // RunContext contains info about current job type RunContext struct { - Name string - Config *Config - Matrix map[string]interface{} - Run *model.Run - EventJSON string - Env map[string]string - ExtraPath []string - CurrentStep string - StepResults map[string]*stepResult - ExprEval ExpressionEvaluator - JobContainer container.Container + Name string + Config *Config + Matrix map[string]interface{} + Run *model.Run + EventJSON string + Env map[string]string + ExtraPath []string + CurrentStep string + StepResults map[string]*stepResult + ExprEval ExpressionEvaluator + JobContainer container.Container + OutputMappings map[MappableOutput]MappableOutput +} + +type MappableOutput struct { + StepID string + OutputName string } func (rc *RunContext) String() string { @@ -372,10 +378,19 @@ func createContainerName(parts ...string) string { if i == len(parts)-1 { name = append(name, pattern.ReplaceAllString(part, "-")) } else { - name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen)) + // If any part has a '-' on the end it is likely part of a matrix job. + // Let's preserve the number to prevent clashes in container names. + re := regexp.MustCompile("-[0-9]+$") + num := re.FindStringSubmatch(part) + if len(num) > 0 { + name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen-len(num[0]))) + name = append(name, num[0]) + } else { + name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen)) + } } } - return strings.Trim(strings.Join(name, "-"), "-") + return strings.ReplaceAll(strings.Trim(strings.Join(name, "-"), "-"),"--","-") } func trimToLen(s string, l int) string { diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go old mode 100644 new mode 100755 index 3175de7..7e15d0a --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -97,6 +97,7 @@ func TestRunEvent(t *testing.T) { {"testdata", "workdir", "push", "", platforms, "linux/amd64"}, // {"testdata", "issue-228", "push", "", platforms, "linux/amd64"}, // TODO [igni]: Remove this once everything passes {"testdata", "defaults-run", "push", "", platforms, "linux/amd64"}, + {"testdata", "uses-composite", "push", "", platforms, "linux/amd64"}, // linux/arm64 {"testdata", "basic", "push", "", platforms, "linux/arm64"}, @@ -116,6 +117,7 @@ func TestRunEvent(t *testing.T) { {"testdata", "workdir", "push", "", platforms, "linux/arm64"}, // {"testdata", "issue-228", "push", "", platforms, "linux/arm64"}, // TODO [igni]: Remove this once everything passes {"testdata", "defaults-run", "push", "", platforms, "linux/arm64"}, + {"testdata", "uses-composite", "push", "", platforms, "linux/arm64"}, } log.SetLevel(log.DebugLevel) diff --git a/pkg/runner/step_context.go b/pkg/runner/step_context.go old mode 100644 new mode 100755 index 49d1c75..f8bb7bc --- a/pkg/runner/step_context.go +++ b/pkg/runner/step_context.go @@ -489,10 +489,72 @@ func (sc *StepContext) runAction(actionDir string, actionPath string) common.Exe ).Finally( stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), )(ctx) + case model.ActionRunsUsingComposite: + for outputName, output := range action.Outputs { + re := regexp.MustCompile(`\${{ steps\.([a-zA-Z_][a-zA-Z0-9_-]+)\.outputs\.([a-zA-Z_][a-zA-Z0-9_-]+) }}`) + matches := re.FindStringSubmatch(output.Value) + if len(matches) > 2 { + if sc.RunContext.OutputMappings == nil { + sc.RunContext.OutputMappings = make(map[MappableOutput]MappableOutput) + } + + k := MappableOutput{StepID: matches[1], OutputName: matches[2]} + v := MappableOutput{StepID: step.ID, OutputName: outputName} + sc.RunContext.OutputMappings[k] = v + } + } + + var executors []common.Executor + stepID := 0 + for _, compositeStep := range action.Runs.Steps { + stepClone := compositeStep + // Take a copy of the run context structure (rc is a pointer) + // Then take the address of the new structure + rcCloneStr := *rc + rcClone := &rcCloneStr + if stepClone.ID == "" { + stepClone.ID = fmt.Sprintf("composite-%d", stepID) + stepID++ + } + rcClone.CurrentStep = stepClone.ID + + if err := compositeStep.Validate(); err != nil { + return err + } + + // Setup the outputs for the composite steps + if _, ok := rcClone.StepResults[stepClone.ID]; ! ok { + rcClone.StepResults[stepClone.ID] = &stepResult{ + Success: true, + Outputs: make(map[string]string), + } + } + + stepClone.Run = strings.ReplaceAll(stepClone.Run, "${{ github.action_path }}", filepath.Join(containerActionDir, actionName)) + + stepContext := StepContext{ + RunContext: rcClone, + Step: &stepClone, + Env: mergeMaps(sc.Env, stepClone.Env), + } + + // Interpolate the outer inputs into the composite step with items + exprEval := sc.NewExpressionEvaluator() + for k, v := range stepContext.Step.With { + + if strings.Contains(v, "inputs") { + stepContext.Step.With[k] = exprEval.Interpolate(v) + } + } + + executors = append(executors, stepContext.Executor()) + } + return common.NewPipelineExecutor(executors...)(ctx) default: return fmt.Errorf(fmt.Sprintf("The runs.using key must be one of: %v, got %s", []string{ model.ActionRunsUsingDocker, model.ActionRunsUsingNode12, + model.ActionRunsUsingComposite, }, action.Runs.Using)) } } diff --git a/pkg/runner/testdata/uses-composite/composite_action/action.yml b/pkg/runner/testdata/uses-composite/composite_action/action.yml new file mode 100644 index 0000000..6c2b080 --- /dev/null +++ b/pkg/runner/testdata/uses-composite/composite_action/action.yml @@ -0,0 +1,48 @@ +--- +name: "Test Composite Action" +description: "Test action uses composite" + +inputs: + test_input_required: + description: "Required input" + required: true + test_input_optional: + description: "optional defaulted input" + required: false + default: "test_input_optional_value" + +outputs: + test_output: + description: "Output value to pass up" + value: ${{ step.output.outputs.test_output }} + +runs: + using: "composite" + steps: + - run: | + echo "#####################################" + echo "Inputs:" + echo "---" + echo "test_input_required=${{ inputs.test_input_required }}" + echo "test_input_optional=${{ inputs.test_input_optional }}" + echo "---" + shell: bash + + # Let's test the inputs + - run: | + if [ "${{ inputs.test_input_required }}" != "test_input_required_value" ]; then + exit 1 + fi + shell: bash + + - run: | + if [ "${{ inputs.test_input_optional }}" != "test_input_optional_value" ]; then + exit 1 + fi + shell: bash + + # Let's send up an output to test + - run: echo "::set-output name=test_output::test_output_value" + shell: bash + + diff --git a/pkg/runner/testdata/uses-composite/push.yml b/pkg/runner/testdata/uses-composite/push.yml new file mode 100755 index 0000000..f255a5e --- /dev/null +++ b/pkg/runner/testdata/uses-composite/push.yml @@ -0,0 +1,18 @@ +name: uses-docker-url +on: push + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: ./uses-composite/composite_action + id: composite + with: + test_input_required: 'test_input_required_value' + test_input_optional: 'test_input_optional_value' + + - if: steps.composite.outputs.test_output != "test_output_value" + run: | + echo "steps.composite.outputs.test_output=${{ steps.composite.outputs.test_output }}" + exit 1 +