From 8f5918942d58af076d6203bb136708f74618ceb1 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 24 Feb 2020 16:38:49 -0800 Subject: [PATCH] move action cache to volume --- pkg/container/docker_run.go | 90 +++++++++++++++++++++++++++++++++++++ pkg/runner/run_context.go | 11 +++-- pkg/runner/step_context.go | 49 +++++++------------- 3 files changed, 110 insertions(+), 40 deletions(-) diff --git a/pkg/container/docker_run.go b/pkg/container/docker_run.go index f964b41..f7ee069 100644 --- a/pkg/container/docker_run.go +++ b/pkg/container/docker_run.go @@ -6,7 +6,10 @@ import ( "context" "fmt" "io" + "io/ioutil" "os" + "path/filepath" + "strings" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -44,6 +47,7 @@ type FileEntry struct { type Container interface { Create() common.Executor Copy(destPath string, files ...*FileEntry) common.Executor + CopyDir(destPath string, srcPath string) common.Executor Pull(forcePull bool) common.Executor Start(attach bool) common.Executor Exec(command []string, env map[string]string) common.Executor @@ -95,6 +99,14 @@ func (cr *containerReference) Copy(destPath string, files ...*FileEntry) common. ).IfNot(common.Dryrun) } +func (cr *containerReference) CopyDir(destPath string, srcPath string) common.Executor { + return common.NewPipelineExecutor( + cr.connect(), + cr.find(), + cr.copyDir(destPath, srcPath), + ).IfNot(common.Dryrun) +} + func (cr *containerReference) Exec(command []string, env map[string]string) common.Executor { return common.NewPipelineExecutor( @@ -289,6 +301,84 @@ func (cr *containerReference) exec(cmd []string, env map[string]string) common.E } } +func (cr *containerReference) copyDir(dstPath string, srcPath string) common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + tarFile, err := ioutil.TempFile("", "act") + if err != nil { + return err + } + log.Debugf("Writing tarball %s from %s", tarFile.Name(), srcPath) + defer tarFile.Close() + //defer os.Remove(tarFile.Name()) + tw := tar.NewWriter(tarFile) + + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + + err = filepath.Walk(srcPath, func(file string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + // return on non-regular files (thanks to [kumo](https://medium.com/@komuw/just-like-you-did-fbdd7df829d3) for this suggested update) + if !fi.Mode().IsRegular() { + return nil + } + + // create a new dir/file header + header, err := tar.FileInfoHeader(fi, fi.Name()) + if err != nil { + return err + } + + // update the name to correctly reflect the desired destination when untaring + header.Name = strings.TrimPrefix(file, srcPrefix) + + // write the header + if err := tw.WriteHeader(header); err != nil { + return err + } + + // open files for taring + f, err := os.Open(file) + if err != nil { + return err + } + + // copy file data into tar writer + if _, err := io.Copy(tw, f); err != nil { + return err + } + + // manually close here after each file operation; defering would cause each file close + // to wait until all operations have completed. + f.Close() + + return nil + }) + if err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + + logger.Debugf("Extracting content from '%s' to '%s'", tarFile.Name(), dstPath) + _, err = tarFile.Seek(0, 0) + if err != nil { + return errors.WithStack(err) + } + err = cr.cli.CopyToContainer(ctx, cr.id, dstPath, tarFile, types.CopyToContainerOptions{}) + if err != nil { + return errors.WithStack(err) + } + return nil + } +} + func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index a86a009..429db61 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -80,11 +80,6 @@ func (rc *RunContext) startJobContainer() common.Executor { bindModifiers = ":delegated" } - hostActionCache := os.Getenv("ACT_HOST_ACTION_CACHE") - if hostActionCache == "" { - hostActionCache = rc.ActionCacheDir() - } - envList = append(envList, fmt.Sprintf("%s=%s", "ACT_HOST_ACTION_CACHE", hostActionCache)) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/toolcache")) rc.JobContainer = container.NewContainer(&container.NewContainerInput{ @@ -97,11 +92,11 @@ func (rc *RunContext) startJobContainer() common.Executor { Mounts: map[string]string{ name: "/github", "act-toolcache": "/toolcache", + "act-actions": "/actions", }, Binds: []string{ fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers), - fmt.Sprintf("%s:%s%s", hostActionCache, "/github/home/.cache/act", bindModifiers), fmt.Sprintf("%s:%s", "/var/run/docker.sock", "/var/run/docker.sock"), }, Stdout: logWriter, @@ -117,6 +112,10 @@ func (rc *RunContext) startJobContainer() common.Executor { Name: "workflow/event.json", Mode: 644, Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "home/.act", + Mode: 644, + Body: "", }), )(ctx) } diff --git a/pkg/runner/step_context.go b/pkg/runner/step_context.go index a78211c..7d2fd80 100644 --- a/pkg/runner/step_context.go +++ b/pkg/runner/step_context.go @@ -65,7 +65,7 @@ func (sc *StepContext) Executor() common.Executor { } } - actionDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), remoteAction.Repo) + actionDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), strings.ReplaceAll(step.Uses, "/", "-")) return common.NewPipelineExecutor( common.NewGitCloneExecutor(common.NewGitCloneExecutorInput{ URL: remoteAction.CloneURL(), @@ -170,6 +170,7 @@ func (sc *StepContext) newStepContainer(ctx context.Context, image string, cmd [ Mounts: map[string]string{ rc.jobContainerName(): "/github", "act-toolcache": "/toolcache", + "act-actions": "/actions", }, Binds: []string{ fmt.Sprintf("%s:%s%s", rc.Config.Workdir, "/github/workspace", bindModifiers), @@ -234,16 +235,24 @@ func (sc *StepContext) runAction(actionDir string) common.Executor { actionName := "" containerActionDir := "." - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - actionName = strings.TrimPrefix(actionDir, rc.Config.Workdir) + if step.Type() == model.StepTypeUsesActionLocal { + actionName = strings.TrimPrefix(strings.TrimPrefix(actionDir, rc.Config.Workdir), string(filepath.Separator)) containerActionDir = "/github/workspace" - } else if strings.HasPrefix(actionDir, rc.ActionCacheDir()) { - actionName = strings.TrimPrefix(actionDir, rc.ActionCacheDir()) - containerActionDir = "/github/home/.cache/act" + } else if step.Type() == model.StepTypeUsesActionRemote { + actionName = strings.TrimPrefix(strings.TrimPrefix(actionDir, rc.ActionCacheDir()), string(filepath.Separator)) + containerActionDir = "/actions" } + log.Debugf("actionDir=%s Workdir=%s ActionCacheDir=%s actionName=%s containerActionDir=%s", actionDir, rc.Config.Workdir, rc.ActionCacheDir(), actionName, containerActionDir) + switch action.Runs.Using { case model.ActionRunsUsingNode12: + if step.Type() == model.StepTypeUsesActionRemote { + err := rc.JobContainer.CopyDir(containerActionDir+string(filepath.Separator), actionDir)(ctx) + if err != nil { + return err + } + } return rc.execJobContainer([]string{"node", fmt.Sprintf("%s/%s/%s", containerActionDir, actionName, action.Runs.Main)}, sc.Env)(ctx) case model.ActionRunsUsingDocker: var prepImage common.Executor @@ -278,34 +287,6 @@ func (sc *StepContext) runAction(actionDir string) common.Executor { ).Finally( stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), )(ctx) - - /* - case model.ActionRunsUsingNode12: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/workspace/%s/%s", strings.TrimPrefix(actionDir, rc.Config.Workdir), action.Runs.Main) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Entrypoint = fmt.Sprintf("node /github/home/%s/%s", strings.TrimPrefix(actionDir, rc.Tempdir), action.Runs.Main) - } - case model.ActionRunsUsingDocker: - if strings.HasPrefix(actionDir, rc.Config.Workdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Config.Workdir)) - } else if strings.HasPrefix(actionDir, rc.Tempdir) { - containerSpec.Name = rc.createStepContainerName(strings.TrimPrefix(actionDir, rc.Tempdir)) - } - containerSpec.Reuse = rc.Config.ReuseContainers - if strings.HasPrefix(action.Runs.Image, "docker://") { - containerSpec.Image = strings.TrimPrefix(action.Runs.Image, "docker://") - containerSpec.Entrypoint = strings.Join(action.Runs.Entrypoint, " ") - containerSpec.Args = strings.Join(action.Runs.Args, " ") - } else { - containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") - contextDir := filepath.Join(actionDir, action.Runs.Main) - return container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ - ContextDir: contextDir, - ImageTag: containerSpec.Image, - })(ctx) - } - */ } return nil }