Add support for composite actions (#514)

* Add support for composite actions

* Fix to make more complex composite actions work

* Fix to make more complex composite actions work

* Let's validate the steps in the composite steps to fail on uses and run's without shell, like the real world

* Add support for composite actions

* Add workflow to test composite actions

* Log instead of panicing when output is mismatched

* Merge maps so environment variables are not lost

* Remove Debug

* Correect merge error

* Remove invalid composite tests.

* Fix composite test

Co-authored-by: Casey Lee <cplee@nektos.com>
Co-authored-by: monkers <mikem@msquaredconsulting.co.uk>
Co-authored-by: Mike Moncrieffe <69815687+mikemonkers@users.noreply.github.com>

.github/actions/composite/action.yml

@@ -0,0 +1,22 @@
+name: 'Test Composite Action'
+description: 'Test Composite Action'
+inputs:
+  input:
+    description: 'String to echo'
+    required: true
+    default: 'Hello World'
+outputs:
+  output:
+    description: 'Output string'
+    value: ${{ steps.set-output.outputs.string }}
+runs:
+  using: "composite"
+  steps:
+    - run: echo Hello ${{ inputs.input }}.
+      shell: bash
+    - id: set-output
+      run: echo "::set-output name=string::output string"
+      shell: bash
+    - run: |
+        ${{ github.action_path }}/script.sh
+      shell: bash

.github/actions/composite/script.sh

@@ -0,0 +1 @@
+echo "Output from script"

.github/workflows/test-composite.yml

@@ -0,0 +1,25 @@
+name: "Test composite actions"
+on: push
+
+env:
+  ACT: true
+
+jobs:
+  test-composite-actions:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "✅ I should echo default 'Hello World'"
+        id: step0
+        uses: ./.github/actions/composite
+
+      - name: "Check action output"
+        if: steps.step0.outputs.output != 'output string'
+        run: echo "expected output to be 'output string' but it wasn't"
+
+      - name: "✅ I should echo 'Test string'"
+        id: step1
+        uses: ./.github/actions/composite
+        with:
+          input: 'Test string'

pkg/model/action.go

@@ -20,10 +20,11 @@ func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(interface{}) error) error
 	// Force input to lowercase for case insensitive comparison
 	format := ActionRunsUsing(strings.ToLower(using))
 	switch format {
-	case ActionRunsUsingNode12, ActionRunsUsingDocker:
+	case ActionRunsUsingNode12, ActionRunsUsingDocker, ActionRunsUsingComposite:
 		*a = format
 	default:
 		return fmt.Errorf(fmt.Sprintf("The runs.using key in action.yml must be one of: %v, got %s", []string{
+			ActionRunsUsingComposite,
 			ActionRunsUsingDocker,
 			ActionRunsUsingNode12,
 		}, format))
@@ -36,6 +37,8 @@ const (
 	ActionRunsUsingNode12 = "node12"
 	// ActionRunsUsingDocker for running with docker
 	ActionRunsUsingDocker = "docker"
+	// ActionRunsUsingComposite for running composite
+	ActionRunsUsingComposite = "composite"
 )
 
 // ActionRuns are a field in Action
@@ -46,6 +49,7 @@ type ActionRuns struct {
 	Image      string            `yaml:"image"`
 	Entrypoint []string          `yaml:"entrypoint"`
 	Args       []string          `yaml:"args"`
+	Steps      []Step            `yaml:"steps"`
 }
 
 // Action describes a metadata file for GitHub actions. The metadata filename must be either action.yml or action.yaml. The data in the metadata file defines the inputs, outputs and main entrypoint for your action.
@@ -72,6 +76,7 @@ type Input struct {
 // Output parameters allow you to declare data that an action sets. Actions that run later in a workflow can use the output data set in previously run actions. For example, if you had an action that performed the addition of two inputs (x + y = z), the action could output the sum (z) for other actions to use as an input.
 type Output struct {
 	Description string `yaml:"description"`
+	Value       string `yaml:"value"`
 }
 
 // ReadAction reads an action from a reader

pkg/model/workflow.go

@@ -312,6 +312,15 @@ func (s *Step) Type() StepType {
 	return StepTypeUsesActionRemote
 }
 
+func (s *Step) Validate() error {
+    if s.Type() != StepTypeRun {
+        return fmt.Errorf("(StepID: %s): Unexpected value 'uses'", s.String())
+    } else if s.Shell == "" {
+        return fmt.Errorf("(StepID: %s): Required property is missing: 'shell'", s.String())
+    }
+    return nil
+}
+
 // ReadWorkflow returns a list of jobs for a given workflow file reader
 func ReadWorkflow(in io.Reader) (*Workflow, error) {
 	w := new(Workflow)

pkg/runner/command.go

@@ -77,8 +77,21 @@ func (rc *RunContext) setEnv(ctx context.Context, kvPairs map[string]string, arg
 	rc.Env[kvPairs["name"]] = arg
 }
 func (rc *RunContext) setOutput(ctx context.Context, kvPairs map[string]string, arg string) {
-	common.Logger(ctx).Infof("  \U00002699  ::set-output:: %s=%s", kvPairs["name"], arg)
+	stepID := rc.CurrentStep
-	rc.StepResults[rc.CurrentStep].Outputs[kvPairs["name"]] = arg
+	outputName := kvPairs["name"]
+	if outputMapping, ok := rc.OutputMappings[MappableOutput{StepID: stepID, OutputName: outputName}]; ok {
+		stepID = outputMapping.StepID
+		outputName = outputMapping.OutputName
+	}
+
+	result, ok := rc.StepResults[stepID]
+	if !ok {
+		common.Logger(ctx).Infof("  \U00002757  no outputs used step '%s'", stepID)
+		return
+	}
+
+	common.Logger(ctx).Infof("  \U00002699  ::set-output:: %s=%s", outputName, arg)
+	result.Outputs[outputName] = arg
 }
 func (rc *RunContext) addPath(ctx context.Context, arg string) {
 	common.Logger(ctx).Infof("  \U00002699  ::add-path:: %s", arg)

pkg/runner/run_context.go

@@ -20,17 +20,23 @@ import (
 
 // RunContext contains info about current job
 type RunContext struct {
-	Name         string
+	Name           string
-	Config       *Config
+	Config         *Config
-	Matrix       map[string]interface{}
+	Matrix         map[string]interface{}
-	Run          *model.Run
+	Run            *model.Run
-	EventJSON    string
+	EventJSON      string
-	Env          map[string]string
+	Env            map[string]string
-	ExtraPath    []string
+	ExtraPath      []string
-	CurrentStep  string
+	CurrentStep    string
-	StepResults  map[string]*stepResult
+	StepResults    map[string]*stepResult
-	ExprEval     ExpressionEvaluator
+	ExprEval       ExpressionEvaluator
-	JobContainer container.Container
+	JobContainer   container.Container
+	OutputMappings map[MappableOutput]MappableOutput
+}
+
+type MappableOutput struct {
+	StepID     string
+	OutputName string
 }
 
 func (rc *RunContext) String() string {
@@ -372,10 +378,19 @@ func createContainerName(parts ...string) string {
 		if i == len(parts)-1 {
 			name = append(name, pattern.ReplaceAllString(part, "-"))
 		} else {
-			name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen))
+		    // If any part has a '-<number>' on the end it is likely part of a matrix job.
+		    // Let's preserve the number to prevent clashes in container names.
+            re := regexp.MustCompile("-[0-9]+$")
+            num := re.FindStringSubmatch(part)
+            if len(num) > 0 {
+		        name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen-len(num[0])))
+    			name = append(name, num[0])
+            } else {
+			    name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen))
+			}
 		}
 	}
-	return strings.Trim(strings.Join(name, "-"), "-")
+	return strings.ReplaceAll(strings.Trim(strings.Join(name, "-"), "-"),"--","-")
 }
 
 func trimToLen(s string, l int) string {

pkg/runner/runner_test.go

@@ -97,6 +97,7 @@ func TestRunEvent(t *testing.T) {
 		{"testdata", "workdir", "push", "", platforms, "linux/amd64"},
 		// {"testdata", "issue-228", "push", "", platforms, "linux/amd64"}, // TODO [igni]: Remove this once everything passes
 		{"testdata", "defaults-run", "push", "", platforms, "linux/amd64"},
+		{"testdata", "uses-composite", "push", "", platforms, "linux/amd64"},
 
 		// linux/arm64
 		{"testdata", "basic", "push", "", platforms, "linux/arm64"},
@@ -116,6 +117,7 @@ func TestRunEvent(t *testing.T) {
 		{"testdata", "workdir", "push", "", platforms, "linux/arm64"},
 		// {"testdata", "issue-228", "push", "", platforms, "linux/arm64"}, // TODO [igni]: Remove this once everything passes
 		{"testdata", "defaults-run", "push", "", platforms, "linux/arm64"},
+		{"testdata", "uses-composite", "push", "", platforms, "linux/arm64"},
 	}
 	log.SetLevel(log.DebugLevel)
 

pkg/runner/step_context.go

@@ -489,10 +489,72 @@ func (sc *StepContext) runAction(actionDir string, actionPath string) common.Exe
 			).Finally(
 				stepContainer.Remove().IfBool(!rc.Config.ReuseContainers),
 			)(ctx)
+		case model.ActionRunsUsingComposite:
+			for outputName, output := range action.Outputs {
+				re := regexp.MustCompile(`\${{ steps\.([a-zA-Z_][a-zA-Z0-9_-]+)\.outputs\.([a-zA-Z_][a-zA-Z0-9_-]+) }}`)
+				matches := re.FindStringSubmatch(output.Value)
+				if len(matches) > 2 {
+					if sc.RunContext.OutputMappings == nil {
+						sc.RunContext.OutputMappings = make(map[MappableOutput]MappableOutput)
+					}
+
+					k := MappableOutput{StepID: matches[1], OutputName: matches[2]}
+					v := MappableOutput{StepID: step.ID, OutputName: outputName}
+					sc.RunContext.OutputMappings[k] = v
+				}
+			}
+
+			var executors []common.Executor
+			stepID := 0
+			for _, compositeStep := range action.Runs.Steps {
+				stepClone := compositeStep
+                // Take a copy of the run context structure (rc is a pointer)
+                // Then take the address of the new structure
+                rcCloneStr := *rc
+                rcClone := &rcCloneStr
+				if stepClone.ID == "" {
+					stepClone.ID = fmt.Sprintf("composite-%d", stepID)
+					stepID++
+				}
+                rcClone.CurrentStep = stepClone.ID
+
+                if err := compositeStep.Validate(); err != nil {
+                    return err
+                }
+
+                // Setup the outputs for the composite steps
+                if _, ok := rcClone.StepResults[stepClone.ID]; ! ok {
+                    rcClone.StepResults[stepClone.ID]  = &stepResult{
+                        Success: true,
+                        Outputs: make(map[string]string),
+                    }
+                }
+
+				stepClone.Run = strings.ReplaceAll(stepClone.Run, "${{ github.action_path }}", filepath.Join(containerActionDir, actionName))
+
+				stepContext := StepContext{
+					RunContext: rcClone,
+					Step:       &stepClone,
+					Env:        mergeMaps(sc.Env, stepClone.Env),
+				}
+
+                // Interpolate the outer inputs into the composite step with items
+                exprEval := sc.NewExpressionEvaluator()
+                for k, v := range stepContext.Step.With {
+
+                    if strings.Contains(v, "inputs") {
+                        stepContext.Step.With[k] = exprEval.Interpolate(v)
+                    }
+                }
+
+				executors = append(executors, stepContext.Executor())
+			}
+			return common.NewPipelineExecutor(executors...)(ctx)
 		default:
 			return fmt.Errorf(fmt.Sprintf("The runs.using key must be one of: %v, got %s", []string{
 				model.ActionRunsUsingDocker,
 				model.ActionRunsUsingNode12,
+				model.ActionRunsUsingComposite,
 			}, action.Runs.Using))
 		}
 	}

pkg/runner/testdata/uses-composite/composite_action/action.yml

@@ -0,0 +1,48 @@
+---
+name: "Test Composite Action"
+description: "Test action uses composite"
+
+inputs:
+  test_input_required:
+    description: "Required input"
+    required: true
+  test_input_optional:
+    description: "optional defaulted input"
+    required: false
+    default: "test_input_optional_value"
+
+outputs:
+  test_output:
+    description: "Output value to pass up"
+    value: ${{ step.output.outputs.test_output }}
+
+runs:
+  using: "composite"
+  steps:
+    - run: |
+        echo "#####################################"
+        echo "Inputs:"
+        echo "---"
+        echo "test_input_required=${{ inputs.test_input_required }}"
+        echo "test_input_optional=${{ inputs.test_input_optional }}"
+        echo "---"
+      shell: bash
+
+    # Let's test the inputs
+    - run: |
+        if [ "${{ inputs.test_input_required }}" != "test_input_required_value" ]; then
+          exit 1
+        fi
+      shell: bash
+
+    - run: |
+        if [ "${{ inputs.test_input_optional }}" != "test_input_optional_value" ]; then
+          exit 1
+        fi
+      shell: bash
+
+    # Let's send up an output to test
+    - run: echo "::set-output name=test_output::test_output_value"
+      shell: bash
+
+

pkg/runner/testdata/uses-composite/push.yml

@@ -0,0 +1,18 @@
+name: uses-docker-url
+on: push
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: ./uses-composite/composite_action
+      id: composite
+      with:
+        test_input_required: 'test_input_required_value'
+        test_input_optional: 'test_input_optional_value'
+
+    - if: steps.composite.outputs.test_output != "test_output_value"
+      run: |
+        echo "steps.composite.outputs.test_output=${{ steps.composite.outputs.test_output }}"
+        exit 1
+