Separate Container Workdir from host Workdir (#635)

* Separate Container Workdir from Host Workdir

* Add delegated component to MacOS Test

* Lint: Remove leading newline

* Fix trailing path issue
This commit is contained in:
Justin Grote 2021-05-04 14:50:35 -07:00 committed by GitHub
parent 020d6a6083
commit 0f049426f6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 263 additions and 66 deletions

4
.gitignore vendored
View file

@ -25,3 +25,7 @@ pkg/runner/act/
dist/local/act
coverage.txt
.env
#Store your GITHUB_TOKEN secret here for purposes of local testing of actions/checkout and others
.secrets

View file

@ -62,6 +62,32 @@ func (rc *RunContext) jobContainerName() string {
return createContainerName("act", rc.String())
}
// Returns the binds and mounts for the container, resolving paths as appopriate
func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) {
name := rc.jobContainerName()
binds := []string{
fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"),
}
mounts := map[string]string{
"act-toolcache": "/toolcache",
"act-actions": "/actions",
}
if rc.Config.BindWorkdir {
bindModifiers := ""
if runtime.GOOS == "darwin" {
bindModifiers = ":delegated"
}
binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, rc.Config.ContainerWorkdir(), bindModifiers))
} else {
mounts[name] = rc.Config.ContainerWorkdir()
}
return binds, mounts
}
func (rc *RunContext) startJobContainer() common.Executor {
image := rc.platformImage()
@ -80,34 +106,21 @@ func (rc *RunContext) startJobContainer() common.Executor {
name := rc.jobContainerName()
envList := make([]string, 0)
bindModifiers := ""
if runtime.GOOS == "darwin" {
bindModifiers = ":delegated"
}
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/opt/hostedtoolcache"))
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_OS", "Linux"))
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp"))
binds := []string{
fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"),
}
if rc.Config.BindWorkdir {
binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, rc.Config.Workdir, bindModifiers))
}
binds, mounts := rc.GetBindsAndMounts()
rc.JobContainer = container.NewContainer(&container.NewContainerInput{
Cmd: nil,
Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"},
WorkingDir: rc.Config.Workdir,
Image: image,
Name: name,
Env: envList,
Mounts: map[string]string{
name: filepath.Dir(rc.Config.Workdir),
"act-toolcache": "/toolcache",
"act-actions": "/actions",
},
Cmd: nil,
Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"},
WorkingDir: rc.Config.ContainerWorkdir(),
Image: image,
Name: name,
Env: envList,
Mounts: mounts,
NetworkMode: "host",
Binds: binds,
Stdout: logWriter,
@ -121,7 +134,7 @@ func (rc *RunContext) startJobContainer() common.Executor {
var copyToPath string
if !rc.Config.BindWorkdir {
copyToPath, copyWorkspace = rc.localCheckoutPath()
copyToPath = filepath.Join(rc.Config.Workdir, copyToPath)
copyToPath = filepath.Join(rc.Config.ContainerWorkdir(), copyToPath)
}
return common.NewPipelineExecutor(
@ -130,7 +143,7 @@ func (rc *RunContext) startJobContainer() common.Executor {
rc.JobContainer.Create(),
rc.JobContainer.Start(false),
rc.JobContainer.CopyDir(copyToPath, rc.Config.Workdir+string(filepath.Separator)+".", rc.Config.UseGitIgnore).IfBool(copyWorkspace),
rc.JobContainer.Copy(filepath.Dir(rc.Config.Workdir), &container.FileEntry{
rc.JobContainer.Copy(rc.Config.ContainerWorkdir(), &container.FileEntry{
Name: "workflow/event.json",
Mode: 0644,
Body: rc.EventJSON,
@ -163,6 +176,8 @@ func (rc *RunContext) stopJobContainer() common.Executor {
}
}
// Prepare the mounts and binds for the worker
// ActionCacheDir is for rc
func (rc *RunContext) ActionCacheDir() string {
var xdgCache string
@ -468,14 +483,14 @@ func (rc *RunContext) getGithubContext() *githubContext {
}
ghc := &githubContext{
Event: make(map[string]interface{}),
EventPath: fmt.Sprintf("%s/%s", filepath.Dir(rc.Config.Workdir), "workflow/event.json"),
EventPath: fmt.Sprintf("%s/%s", rc.Config.ContainerWorkdir(), "workflow/event.json"),
Workflow: rc.Run.Workflow.Name,
RunID: runID,
RunNumber: runNumber,
Actor: rc.Config.Actor,
EventName: rc.Config.EventName,
Token: token,
Workspace: rc.Config.Workdir,
Workspace: rc.Config.ContainerWorkdir(),
Action: rc.CurrentStep,
}
@ -537,6 +552,10 @@ func (rc *RunContext) getGithubContext() *githubContext {
}
func (ghc *githubContext) isLocalCheckout(step *model.Step) bool {
if step.Type() != model.StepTypeInvalid {
// This will be errored out by the executor later, we need this here to avoid a null panic though
return false
}
if step.Type() != model.StepTypeUsesActionRemote {
return false
}
@ -606,7 +625,7 @@ func withDefaultBranch(b string, event map[string]interface{}) map[string]interf
func (rc *RunContext) withGithubEnv(env map[string]string) map[string]string {
github := rc.getGithubContext()
env["CI"] = "true"
env["GITHUB_ENV"] = fmt.Sprintf("%s/%s", filepath.Dir(rc.Config.Workdir), "workflow/envs.txt")
env["GITHUB_ENV"] = fmt.Sprintf("%s/%s", rc.Config.ContainerWorkdir(), "workflow/envs.txt")
env["GITHUB_WORKFLOW"] = github.Workflow
env["GITHUB_RUN_ID"] = github.RunID
env["GITHUB_RUN_NUMBER"] = github.RunNumber

View file

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"regexp"
"runtime"
"sort"
"strings"
"testing"
@ -211,3 +212,68 @@ jobs:
t.Fatal(err)
}
}
func TestRunContext_GetBindsAndMounts(t *testing.T) {
rctemplate := &RunContext{
Name: "TestRCName",
Run: &model.Run{
Workflow: &model.Workflow{
Name: "TestWorkflowName",
},
},
Config: &Config{
BindWorkdir: false,
},
}
tests := []struct {
windowsPath bool
name string
rc *RunContext
wantbind string
wantmount string
}{
{false, "/mnt/linux", rctemplate, "/mnt/linux", "/mnt/linux"},
{false, "/mnt/path with spaces/linux", rctemplate, "/mnt/path with spaces/linux", "/mnt/path with spaces/linux"},
{true, "C:\\Users\\TestPath\\MyTestPath", rctemplate, "/mnt/c/Users/TestPath/MyTestPath", "/mnt/c/Users/TestPath/MyTestPath"},
{true, "C:\\Users\\Test Path with Spaces\\MyTestPath", rctemplate, "/mnt/c/Users/Test Path with Spaces/MyTestPath", "/mnt/c/Users/Test Path with Spaces/MyTestPath"},
{true, "/LinuxPathOnWindowsShouldFail", rctemplate, "", ""},
}
isWindows := runtime.GOOS == "windows"
for _, testcase := range tests {
// pin for scopelint
testcase := testcase
for _, bindWorkDir := range []bool{true, false} {
// pin for scopelint
bindWorkDir := bindWorkDir
testBindSuffix := ""
if bindWorkDir {
testBindSuffix = "Bind"
}
// Only run windows path tests on windows and non-windows on non-windows
if (isWindows && testcase.windowsPath) || (!isWindows && !testcase.windowsPath) {
t.Run((testcase.name + testBindSuffix), func(t *testing.T) {
config := testcase.rc.Config
config.Workdir = testcase.name
config.BindWorkdir = bindWorkDir
gotbind, gotmount := rctemplate.GetBindsAndMounts()
// Name binds/mounts are either/or
if config.BindWorkdir {
fullBind := testcase.name + ":" + testcase.wantbind
if runtime.GOOS == "darwin" {
fullBind += ":delegated"
}
a.Contains(t, gotbind, fullBind)
} else {
mountkey := testcase.rc.jobContainerName()
a.EqualValues(t, testcase.wantmount, gotmount[mountkey])
}
})
}
}
}
}

View file

@ -4,6 +4,10 @@ import (
"context"
"fmt"
"io/ioutil"
"path/filepath"
"regexp"
"runtime"
"strings"
"github.com/nektos/act/pkg/common"
"github.com/nektos/act/pkg/model"
@ -36,6 +40,46 @@ type Config struct {
UseGitIgnore bool // controls if paths in .gitignore should not be copied into container, default true
}
// Resolves the equivalent host path inside the container
// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
// For use in docker volumes and binds
func (config *Config) containerPath(path string) string {
if runtime.GOOS == "windows" && strings.Contains(path, "/") {
log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
return ""
}
abspath, err := filepath.Abs(path)
if err != nil {
log.Error(err)
return ""
}
// Test if the path is a windows path
windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath)
// Return as-is if no match
if windowsPathComponents == nil {
return abspath
}
// Convert to WSL2-compatible path if it is a windows path
// NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows
// based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work.
driveLetter := strings.ToLower(windowsPathComponents[1])
translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
// Should make something like /mnt/c/Users/person/My Folder/MyActProject
result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`)
return result
}
// Resolves the equivalent host path inside the container
// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
func (config *Config) ContainerWorkdir() string {
return config.containerPath(config.Workdir)
}
type runnerImpl struct {
config *Config
eventJSON string

View file

@ -3,7 +3,10 @@ package runner
import (
"context"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"github.com/joho/godotenv"
@ -40,19 +43,21 @@ type TestJobFileInfo struct {
containerArchitecture string
}
func runTestJobFile(ctx context.Context, t *testing.T, tjfi TestJobFileInfo) {
func runTestJobFile(ctx context.Context, t *testing.T, tjfi TestJobFileInfo, secrets map[string]string) {
t.Run(tjfi.workflowPath, func(t *testing.T) {
workdir, err := filepath.Abs(tjfi.workdir)
assert.NilError(t, err, workdir)
fullWorkflowPath := filepath.Join(workdir, tjfi.workflowPath)
runnerConfig := &Config{
Workdir: workdir,
BindWorkdir: true,
BindWorkdir: false,
EventName: tjfi.eventName,
Platforms: tjfi.platforms,
ReuseContainers: false,
ContainerArchitecture: tjfi.containerArchitecture,
Secrets: secrets,
}
runner, err := New(runnerConfig)
assert.NilError(t, err, tjfi.workflowPath)
@ -106,9 +111,11 @@ func TestRunEvent(t *testing.T) {
log.SetLevel(log.DebugLevel)
ctx := context.Background()
secretspath, _ := filepath.Abs("../../.secrets")
secrets, _ := godotenv.Read(secretspath)
for _, table := range tables {
runTestJobFile(ctx, t, table)
runTestJobFile(ctx, t, table, secrets)
}
}
@ -189,3 +196,60 @@ func TestRunEventPullRequest(t *testing.T) {
err = runner.NewPlanExecutor(plan)(ctx)
assert.NilError(t, err, workflowPath)
}
func TestContainerPath(t *testing.T) {
type containerPathJob struct {
destinationPath string
sourcePath string
workDir string
}
if runtime.GOOS == "windows" {
cwd, err := os.Getwd()
if err != nil {
log.Error(err)
}
rootDrive := os.Getenv("SystemDrive")
rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "")
for _, v := range []containerPathJob{
{"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""},
{"/mnt/f/work/dir", `F:\work\dir`, ""},
{"/mnt/c/windows/to/unix", "windows/to/unix", fmt.Sprintf("%s\\", rootDrive)},
{fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)},
} {
if v.workDir != "" {
if err := os.Chdir(v.workDir); err != nil {
log.Error(err)
t.Fail()
}
}
runnerConfig := &Config{
Workdir: v.sourcePath,
}
assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir))
}
if err := os.Chdir(cwd); err != nil {
log.Error(err)
}
} else {
cwd, err := os.Getwd()
if err != nil {
log.Error(err)
}
for _, v := range []containerPathJob{
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
{"/home/act", `/home/act/`, ""},
{cwd, ".", ""},
} {
runnerConfig := &Config{
Workdir: v.sourcePath,
}
assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir))
}
}
}

View file

@ -165,8 +165,8 @@ func (sc *StepContext) setupShellCommand() common.Executor {
}
scriptName := fmt.Sprintf("workflow/%s", step.ID)
//Reference: https://github.com/actions/runner/blob/8109c962f09d9acc473d92c595ff43afceddb347/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs#L47-L64
//Reference: https://github.com/actions/runner/blob/8109c962f09d9acc473d92c595ff43afceddb347/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs#L19-L27
// Reference: https://github.com/actions/runner/blob/8109c962f09d9acc473d92c595ff43afceddb347/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs#L47-L64
// Reference: https://github.com/actions/runner/blob/8109c962f09d9acc473d92c595ff43afceddb347/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs#L19-L27
runPrepend := ""
runAppend := ""
scriptExt := ""
@ -188,7 +188,7 @@ func (sc *StepContext) setupShellCommand() common.Executor {
run = runPrepend + "\n" + run + "\n" + runAppend
log.Debugf("Wrote command '%s' to '%s'", run, scriptName)
containerPath := fmt.Sprintf("%s/%s", filepath.Dir(rc.Config.Workdir), scriptName)
containerPath := fmt.Sprintf("%s/%s", rc.Config.ContainerWorkdir(), scriptName)
if step.Shell == "" {
step.Shell = rc.Run.Job().Defaults.Run.Shell
@ -204,7 +204,7 @@ func (sc *StepContext) setupShellCommand() common.Executor {
sc.Cmd = strings.Fields(scResolvedCmd)
}
return rc.JobContainer.Copy(fmt.Sprintf("%s/", filepath.Dir(rc.Config.Workdir)), &container.FileEntry{
return rc.JobContainer.Copy(rc.Config.ContainerWorkdir(), &container.FileEntry{
Name: scriptName,
Mode: 0755,
Body: script.String(),
@ -236,34 +236,20 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [
entrypoint[i] = stepEE.Interpolate(v)
}
bindModifiers := ""
if runtime.GOOS == "darwin" {
bindModifiers = ":delegated"
}
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/opt/hostedtoolcache"))
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_OS", "Linux"))
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp"))
binds := []string{
fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"),
}
if rc.Config.BindWorkdir {
binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, rc.Config.Workdir, bindModifiers))
}
binds, mounts := rc.GetBindsAndMounts()
stepContainer := container.NewContainer(&container.NewContainerInput{
Cmd: cmd,
Entrypoint: entrypoint,
WorkingDir: rc.Config.Workdir,
Image: image,
Name: createContainerName(rc.jobContainerName(), step.ID),
Env: envList,
Mounts: map[string]string{
rc.jobContainerName(): filepath.Dir(rc.Config.Workdir),
"act-toolcache": "/toolcache",
"act-actions": "/actions",
},
Cmd: cmd,
Entrypoint: entrypoint,
WorkingDir: rc.Config.ContainerWorkdir(),
Image: image,
Name: createContainerName(rc.jobContainerName(), step.ID),
Env: envList,
Mounts: mounts,
NetworkMode: fmt.Sprintf("container:%s", rc.jobContainerName()),
Binds: binds,
Stdout: logWriter,
@ -371,12 +357,15 @@ func getOsSafeRelativePath(s, prefix string) string {
func (sc *StepContext) getContainerActionPaths(step *model.Step, actionDir string, rc *RunContext) (string, string) {
actionName := ""
containerActionDir := "."
if step.Type() == model.StepTypeUsesActionLocal {
if !rc.Config.BindWorkdir && step.Type() != model.StepTypeUsesActionRemote {
actionName = getOsSafeRelativePath(actionDir, rc.Config.Workdir)
containerActionDir = rc.Config.Workdir
containerActionDir = rc.Config.ContainerWorkdir() + "/_actions/" + actionName
} else if step.Type() == model.StepTypeUsesActionRemote {
actionName = getOsSafeRelativePath(actionDir, rc.ActionCacheDir())
containerActionDir = "/actions"
containerActionDir = rc.Config.ContainerWorkdir() + "/_actions/" + actionName
} else if step.Type() == model.StepTypeUsesActionLocal {
actionName = getOsSafeRelativePath(actionDir, rc.Config.Workdir)
containerActionDir = rc.Config.ContainerWorkdir() + "/_actions/" + actionName
}
if actionName == "" {
@ -388,6 +377,7 @@ func (sc *StepContext) getContainerActionPaths(step *model.Step, actionDir strin
return actionName, containerActionDir
}
// nolint: gocyclo
func (sc *StepContext) runAction(actionDir string, actionPath string) common.Executor {
rc := sc.RunContext
step := sc.Step
@ -402,7 +392,13 @@ func (sc *StepContext) runAction(actionDir string, actionPath string) common.Exe
}
}
actionName, containerActionDir := sc.getContainerActionPaths(step, actionDir, rc)
actionLocation := ""
if actionPath != "" {
actionLocation = path.Join(actionDir, actionPath)
} else {
actionLocation = actionDir
}
actionName, containerActionDir := sc.getContainerActionPaths(step, actionLocation, rc)
sc.Env = mergeMaps(sc.Env, action.Runs.Env)
@ -410,13 +406,16 @@ func (sc *StepContext) runAction(actionDir string, actionPath string) common.Exe
maybeCopyToActionDir := func() error {
if step.Type() != model.StepTypeUsesActionRemote {
return nil
// If the workdir is bound to our repository then we don't need to copy the file
if rc.Config.BindWorkdir {
return nil
}
}
err := removeGitIgnore(actionDir)
if err != nil {
return err
}
return rc.JobContainer.CopyDir(containerActionDir+"/", actionDir, rc.Config.UseGitIgnore)(ctx)
return rc.JobContainer.CopyDir(containerActionDir+"/", actionLocation+"/", rc.Config.UseGitIgnore)(ctx)
}
switch action.Runs.Using {
@ -425,7 +424,7 @@ func (sc *StepContext) runAction(actionDir string, actionPath string) common.Exe
if err != nil {
return err
}
containerArgs := []string{"node", path.Join(containerActionDir, actionName, actionPath, action.Runs.Main)}
containerArgs := []string{"node", path.Join(containerActionDir, action.Runs.Main)}
log.Debugf("executing remote job container: %s", containerArgs)
return rc.execJobContainer(containerArgs, sc.Env)(ctx)
case model.ActionRunsUsingDocker:

View file

@ -2,8 +2,10 @@ package runner
import (
"context"
"path/filepath"
"testing"
"github.com/joho/godotenv"
"github.com/nektos/act/pkg/common"
)
@ -20,7 +22,8 @@ func TestStepContextExecutor(t *testing.T) {
}
// These tests are sufficient to only check syntax.
ctx := common.WithDryrun(context.Background(), true)
secrets, _ := godotenv.Read(filepath.Join("..", ".secrets"))
for _, table := range tables {
runTestJobFile(ctx, t, table)
runTestJobFile(ctx, t, table, secrets)
}
}

View file

@ -5,13 +5,11 @@ jobs:
workdir:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: ls -alFt "${GITHUB_WORKSPACE}/workdir"
- run: mkdir -p "${GITHUB_WORKSPACE}/workdir"
- run: '[[ "$(pwd)" == "${GITHUB_WORKSPACE}/workdir" ]]'
working-directory: workdir
noworkdir:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: '[[ "$(pwd)" == "${GITHUB_WORKSPACE}" ]]'