Compare commits
5 commits
main
...
2023-10-04
Author | SHA1 | Date | |
---|---|---|---|
|
1274b349f1 | ||
|
0c2c019584 | ||
|
1a3160510b | ||
|
0b4fe4cce2 | ||
|
e758d8cc3d |
9 changed files with 682 additions and 10 deletions
44
.forgejo/workflows/test.yml
Normal file
44
.forgejo/workflows/test.yml
Normal file
|
@ -0,0 +1,44 @@
|
||||||
|
name: checks
|
||||||
|
on:
|
||||||
|
- push
|
||||||
|
- pull_request
|
||||||
|
|
||||||
|
env:
|
||||||
|
GOPROXY: https://goproxy.io,direct
|
||||||
|
GOPATH: /go_path
|
||||||
|
GOCACHE: /go_cache
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lint:
|
||||||
|
name: check and test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: cache go path
|
||||||
|
id: cache-go-path
|
||||||
|
uses: https://github.com/actions/cache@v3
|
||||||
|
with:
|
||||||
|
path: /go_path
|
||||||
|
key: go_path-${{ github.repository }}-${{ github.ref_name }}
|
||||||
|
restore-keys: |
|
||||||
|
go_path-${{ github.repository }}-
|
||||||
|
go_path-
|
||||||
|
- name: cache go cache
|
||||||
|
id: cache-go-cache
|
||||||
|
uses: https://github.com/actions/cache@v3
|
||||||
|
with:
|
||||||
|
path: /go_cache
|
||||||
|
key: go_cache-${{ github.repository }}-${{ github.ref_name }}
|
||||||
|
restore-keys: |
|
||||||
|
go_cache-${{ github.repository }}-
|
||||||
|
go_cache-
|
||||||
|
- uses: actions/setup-go@v3
|
||||||
|
with:
|
||||||
|
go-version: 1.20
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
- name: vet checks
|
||||||
|
run: go vet -v ./...
|
||||||
|
- name: build
|
||||||
|
run: go build -v ./...
|
||||||
|
- name: test
|
||||||
|
run: go test -v ./pkg/jobparser
|
||||||
|
# TODO test more packages
|
|
@ -5,6 +5,8 @@ import "context"
|
||||||
type ExecutionsEnvironment interface {
|
type ExecutionsEnvironment interface {
|
||||||
Container
|
Container
|
||||||
ToContainerPath(string) string
|
ToContainerPath(string) string
|
||||||
|
GetName() string
|
||||||
|
GetRoot() string
|
||||||
GetActPath() string
|
GetActPath() string
|
||||||
GetPathVariableName() string
|
GetPathVariableName() string
|
||||||
DefaultPathVariable() string
|
DefaultPathVariable() string
|
||||||
|
|
|
@ -25,16 +25,18 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type HostEnvironment struct {
|
type HostEnvironment struct {
|
||||||
|
Name string
|
||||||
Path string
|
Path string
|
||||||
TmpDir string
|
TmpDir string
|
||||||
ToolCache string
|
ToolCache string
|
||||||
Workdir string
|
Workdir string
|
||||||
ActPath string
|
ActPath string
|
||||||
|
Root string
|
||||||
CleanUp func()
|
CleanUp func()
|
||||||
StdOut io.Writer
|
StdOut io.Writer
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) Create(_ []string, _ []string) common.Executor {
|
func (e *HostEnvironment) Create(_, _ []string) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -66,7 +68,7 @@ func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Exec
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor {
|
func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
srcPrefix := filepath.Dir(srcPath)
|
srcPrefix := filepath.Dir(srcPath)
|
||||||
|
@ -260,7 +262,7 @@ func getEnvListFromMap(env map[string]string) []string {
|
||||||
return envList
|
return envList
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, _, workdir string) error {
|
func (e *HostEnvironment) exec(ctx context.Context, commandparam []string, cmdline string, env map[string]string, user, workdir string) error {
|
||||||
envList := getEnvListFromMap(env)
|
envList := getEnvListFromMap(env)
|
||||||
var wd string
|
var wd string
|
||||||
if workdir != "" {
|
if workdir != "" {
|
||||||
|
@ -272,6 +274,19 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||||
} else {
|
} else {
|
||||||
wd = e.Path
|
wd = e.Path
|
||||||
}
|
}
|
||||||
|
if _, err := os.Stat(wd); err != nil {
|
||||||
|
common.Logger(ctx).Debugf("Failed to stat working directory %s %v\n", wd, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
command := make([]string, len(commandparam))
|
||||||
|
copy(command, commandparam)
|
||||||
|
if user == "root" {
|
||||||
|
command = append([]string{"/usr/bin/sudo"}, command...)
|
||||||
|
} else {
|
||||||
|
common.Logger(ctx).Debugf("lxc-attach --name %v %v", e.Name, command)
|
||||||
|
command = append([]string{"/usr/bin/sudo", "--preserve-env", "--preserve-env=PATH", "/usr/bin/lxc-attach", "--keep-env", "--name", e.Name, "--"}, command...)
|
||||||
|
}
|
||||||
|
|
||||||
f, err := lookupPathHost(command[0], env, e.StdOut)
|
f, err := lookupPathHost(command[0], env, e.StdOut)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -314,7 +329,7 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||||
}
|
}
|
||||||
err = cmd.Run()
|
err = cmd.Run()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("RUN %w", err)
|
||||||
}
|
}
|
||||||
if tty != nil {
|
if tty != nil {
|
||||||
writer.AutoStop = true
|
writer.AutoStop = true
|
||||||
|
@ -367,6 +382,14 @@ func (e *HostEnvironment) ToContainerPath(path string) string {
|
||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (e *HostEnvironment) GetName() string {
|
||||||
|
return e.Name
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *HostEnvironment) GetRoot() string {
|
||||||
|
return e.Root
|
||||||
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) GetActPath() string {
|
func (e *HostEnvironment) GetActPath() string {
|
||||||
actPath := e.ActPath
|
actPath := e.ActPath
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
|
@ -426,7 +449,7 @@ func (e *HostEnvironment) GetRunnerContext(_ context.Context) map[string]interfa
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *HostEnvironment) ReplaceLogWriter(stdout io.Writer, _ io.Writer) (io.Writer, io.Writer) {
|
func (e *HostEnvironment) ReplaceLogWriter(stdout, _ io.Writer) (io.Writer, io.Writer) {
|
||||||
org := e.StdOut
|
org := e.StdOut
|
||||||
e.StdOut = stdout
|
e.StdOut = stdout
|
||||||
return org, org
|
return org, org
|
||||||
|
|
|
@ -10,8 +10,7 @@ import (
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
type LinuxContainerEnvironmentExtensions struct {
|
type LinuxContainerEnvironmentExtensions struct{}
|
||||||
}
|
|
||||||
|
|
||||||
// Resolves the equivalent host path inside the container
|
// Resolves the equivalent host path inside the container
|
||||||
// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
|
// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
|
||||||
|
@ -47,6 +46,14 @@ func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (*LinuxContainerEnvironmentExtensions) GetName() string {
|
||||||
|
return "NAME"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*LinuxContainerEnvironmentExtensions) GetRoot() string {
|
||||||
|
return "/var/run"
|
||||||
|
}
|
||||||
|
|
||||||
func (*LinuxContainerEnvironmentExtensions) GetActPath() string {
|
func (*LinuxContainerEnvironmentExtensions) GetActPath() string {
|
||||||
return "/var/run/act"
|
return "/var/run/act"
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,6 +42,9 @@ var trampoline embed.FS
|
||||||
func readActionImpl(ctx context.Context, step *model.Step, actionDir string, actionPath string, readFile actionYamlReader, writeFile fileWriter) (*model.Action, error) {
|
func readActionImpl(ctx context.Context, step *model.Step, actionDir string, actionPath string, readFile actionYamlReader, writeFile fileWriter) (*model.Action, error) {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
reader, closer, err := readFile("action.yml")
|
reader, closer, err := readFile("action.yml")
|
||||||
|
if err != nil {
|
||||||
|
logger.Debugf("readActionImpl actionDir %s actionPath %s failed %v", actionDir, actionPath, err)
|
||||||
|
}
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
reader, closer, err = readFile("action.yaml")
|
reader, closer, err = readFile("action.yaml")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -439,11 +442,15 @@ func getContainerActionPaths(step *model.Step, actionDir string, rc *RunContext)
|
||||||
actionName = strings.ReplaceAll(actionName, "\\", "/")
|
actionName = strings.ReplaceAll(actionName, "\\", "/")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
common.Logger(context.Background()).Debugf("getContainerActionPaths step.Type %s, rc.Config.Workdir %s, actionName %s, containerActionDir %s", step.Type().String(), rc.Config.Workdir, actionName, containerActionDir)
|
||||||
return actionName, containerActionDir
|
return actionName, containerActionDir
|
||||||
}
|
}
|
||||||
|
|
||||||
func getOsSafeRelativePath(s, prefix string) string {
|
func getOsSafeRelativePath(s, prefix string) string {
|
||||||
actionName := strings.TrimPrefix(s, prefix)
|
actionName := strings.TrimPrefix(s, prefix)
|
||||||
|
if s == actionName {
|
||||||
|
common.Logger(context.Background()).Errorf("getOsSafeRelativePath %s does not beging with %s", s, prefix)
|
||||||
|
}
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
actionName = strings.ReplaceAll(actionName, "\\", "/")
|
actionName = strings.ReplaceAll(actionName, "\\", "/")
|
||||||
}
|
}
|
||||||
|
|
311
pkg/runner/lxc-helpers-lib.sh
Executable file
311
pkg/runner/lxc-helpers-lib.sh
Executable file
|
@ -0,0 +1,311 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
export DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
LXC_SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||||
|
LXC_BIN=/usr/local/bin
|
||||||
|
|
||||||
|
: ${LXC_SUDO:=}
|
||||||
|
: ${LXC_CONTAINER_RELEASE:=bookworm}
|
||||||
|
: ${LXC_HOME:=/home}
|
||||||
|
: ${LXC_VERBOSE:=false}
|
||||||
|
|
||||||
|
source /etc/os-release
|
||||||
|
|
||||||
|
function lxc_release() {
|
||||||
|
echo $VERSION_CODENAME
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_template_release() {
|
||||||
|
echo lxc-helpers-$LXC_CONTAINER_RELEASE
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_root() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
echo /var/lib/lxc/$name/rootfs
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_config() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
echo /var/lib/lxc/$name/config
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_run() {
|
||||||
|
local name="$1"
|
||||||
|
shift
|
||||||
|
|
||||||
|
$LXC_SUDO lxc-attach --clear-env --name $name -- "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_run_script_as() {
|
||||||
|
local name="$1"
|
||||||
|
local user="$2"
|
||||||
|
local script="$3"
|
||||||
|
|
||||||
|
$LXC_SUDO chmod +x $(lxc_root $name)$script
|
||||||
|
$LXC_SUDO lxc-attach --name $name -- sudo --user $user $script
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_run_script() {
|
||||||
|
local name="$1"
|
||||||
|
local script="$2"
|
||||||
|
|
||||||
|
$LXC_SUDO chmod +x $(lxc_root $name)$script
|
||||||
|
lxc_container_run $name $script
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_inside() {
|
||||||
|
local name="$1"
|
||||||
|
shift
|
||||||
|
|
||||||
|
lxc_container_run $name $LXC_BIN/lxc-helpers.sh "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_user_install() {
|
||||||
|
local name="$1"
|
||||||
|
local user_id="$2"
|
||||||
|
local user="$3"
|
||||||
|
|
||||||
|
if test "$user" = root ; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
local root=$(lxc_root $name)
|
||||||
|
|
||||||
|
if ! $LXC_SUDO grep --quiet "^$user " $root/etc/sudoers ; then
|
||||||
|
$LXC_SUDO tee $root/usr/local/bin/lxc-helpers-create-user.sh > /dev/null <<EOF
|
||||||
|
#!/bin/bash
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
mkdir -p $LXC_HOME
|
||||||
|
useradd --base-dir $LXC_HOME --create-home --shell /bin/bash --uid $user_id $user
|
||||||
|
for group in docker kvm libvirt ; do
|
||||||
|
if grep --quiet \$group /etc/group ; then adduser $user \$group ; fi
|
||||||
|
done
|
||||||
|
echo "$user ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
|
||||||
|
sudo --user $user ssh-keygen -b 2048 -N '' -f $LXC_HOME/$user/.ssh/id_rsa
|
||||||
|
EOF
|
||||||
|
lxc_container_run_script $name /usr/local/bin/lxc-helpers-create-user.sh
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_maybe_sudo() {
|
||||||
|
if test $(id -u) != 0 ; then
|
||||||
|
LXC_SUDO=sudo
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_prepare_environment() {
|
||||||
|
lxc_maybe_sudo
|
||||||
|
if ! $(which lxc-create > /dev/null) ; then
|
||||||
|
$LXC_SUDO apt-get install -y -qq make git libvirt0 libpam-cgfs bridge-utils uidmap dnsmasq-base dnsmasq dnsmasq-utils qemu-user-static
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_configure() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
$LXC_SUDO tee -a $(lxc_config $name) > /dev/null <<'EOF'
|
||||||
|
security.nesting = true
|
||||||
|
lxc.cap.drop =
|
||||||
|
lxc.apparmor.profile = unconfined
|
||||||
|
#
|
||||||
|
# /dev/net (docker won't work without /dev/net/tun)
|
||||||
|
#
|
||||||
|
lxc.cgroup2.devices.allow = c 10:200 rwm
|
||||||
|
lxc.mount.entry = /dev/net dev/net none bind,create=dir 0 0
|
||||||
|
#
|
||||||
|
# /dev/kvm (libvirt / kvm won't work without /dev/kvm)
|
||||||
|
#
|
||||||
|
lxc.cgroup2.devices.allow = c 10:232 rwm
|
||||||
|
lxc.mount.entry = /dev/kvm dev/kvm none bind,create=file 0 0
|
||||||
|
#
|
||||||
|
# /dev/loop
|
||||||
|
#
|
||||||
|
lxc.cgroup2.devices.allow = c 10:237 rwm
|
||||||
|
lxc.cgroup2.devices.allow = b 7:* rwm
|
||||||
|
lxc.mount.entry = /dev/loop-control dev/loop-control none bind,create=file 0 0
|
||||||
|
#
|
||||||
|
# /dev/mapper
|
||||||
|
#
|
||||||
|
lxc.cgroup2.devices.allow = c 10:236 rwm
|
||||||
|
lxc.mount.entry = /dev/mapper dev/mapper none bind,create=dir 0 0
|
||||||
|
#
|
||||||
|
# /dev/fuse
|
||||||
|
#
|
||||||
|
lxc.cgroup2.devices.allow = b 10:229 rwm
|
||||||
|
lxc.mount.entry = /dev/fuse dev/fuse none bind,create=file 0 0
|
||||||
|
EOF
|
||||||
|
|
||||||
|
|
||||||
|
#
|
||||||
|
# Wait for the network to come up
|
||||||
|
#
|
||||||
|
local wait_networking=$(lxc_root $name)/usr/local/bin/lxc-helpers-wait-networking.sh
|
||||||
|
$LXC_SUDO tee $wait_networking > /dev/null <<'EOF'
|
||||||
|
#!/bin/sh -e
|
||||||
|
for d in $(seq 60); do
|
||||||
|
getent hosts wikipedia.org > /dev/null && break
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
getent hosts wikipedia.org > /dev/null || getent hosts wikipedia.org
|
||||||
|
EOF
|
||||||
|
$LXC_SUDO chmod +x $wait_networking
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_create() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
lxc_prepare_environment
|
||||||
|
lxc_build_template $(lxc_template_release) "$name"
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_mount() {
|
||||||
|
local name="$1"
|
||||||
|
local dir="$2"
|
||||||
|
|
||||||
|
local config=$(lxc_config $name)
|
||||||
|
|
||||||
|
if ! $LXC_SUDO grep --quiet "lxc.mount.entry = $dir" $config ; then
|
||||||
|
local relative_dir=${dir##/}
|
||||||
|
$LXC_SUDO tee -a $config > /dev/null <<< "lxc.mount.entry = $dir $relative_dir none bind,create=dir 0 0"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function lxc_container_start() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
if lxc_running $name ; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
local logs
|
||||||
|
if $LXC_VERBOSE; then
|
||||||
|
logs="--logfile=/dev/tty"
|
||||||
|
fi
|
||||||
|
|
||||||
|
$LXC_SUDO lxc-start $logs $name
|
||||||
|
$LXC_SUDO lxc-wait --name $name --state RUNNING
|
||||||
|
lxc_container_run $name /usr/local/bin/lxc-helpers-wait-networking.sh
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_stop() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
$LXC_SUDO lxc-ls -1 --running --filter="^$name" | while read container ; do
|
||||||
|
$LXC_SUDO lxc-stop --kill --name="$container"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_container_destroy() {
|
||||||
|
local name="$1"
|
||||||
|
local root="$2"
|
||||||
|
|
||||||
|
if lxc_exists "$name" ; then
|
||||||
|
lxc_container_stop $name $root
|
||||||
|
$LXC_SUDO lxc-destroy --force --name="$name"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_exists() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
test "$($LXC_SUDO lxc-ls --filter=^$name\$)"
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_running() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
test "$($LXC_SUDO lxc-ls --running --filter=^$name\$)"
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_build_template_release() {
|
||||||
|
local name="$(lxc_template_release)"
|
||||||
|
|
||||||
|
if lxc_exists $name ; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
local root=$(lxc_root $name)
|
||||||
|
local packages="sudo,git,python3"
|
||||||
|
$LXC_SUDO lxc-create --name $name --template debian -- --release=$LXC_CONTAINER_RELEASE --packages="$packages"
|
||||||
|
$LXC_SUDO cp -a $LXC_SELF_DIR/lxc-helpers*.sh $root/$LXC_BIN
|
||||||
|
lxc_container_configure $name
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_build_template() {
|
||||||
|
local name="$1"
|
||||||
|
local newname="$2"
|
||||||
|
|
||||||
|
if lxc_exists $newname ; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
if test "$name" = "$(lxc_template_release)" ; then
|
||||||
|
lxc_build_template_release
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! $LXC_SUDO lxc-copy --name=$name --newname=$newname ; then
|
||||||
|
echo lxc-copy --name=$name --newname=$newname failed
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_apt_install() {
|
||||||
|
local name="$1"
|
||||||
|
shift
|
||||||
|
|
||||||
|
lxc_container_inside $name lxc_apt_install_inside "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_apt_install_inside() {
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_install_lxc() {
|
||||||
|
local name="$1"
|
||||||
|
local prefix="$2"
|
||||||
|
|
||||||
|
lxc_container_inside $name lxc_install_lxc_inside $prefix
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_install_lxc_inside() {
|
||||||
|
local prefix="$1"
|
||||||
|
|
||||||
|
local packages="make git libvirt0 libpam-cgfs bridge-utils uidmap dnsmasq-base dnsmasq dnsmasq-utils qemu-user-static lxc-templates debootstrap"
|
||||||
|
if test "$(lxc_release)" = bookworm ; then
|
||||||
|
packages="$packages distro-info"
|
||||||
|
fi
|
||||||
|
|
||||||
|
lxc_apt_install_inside $packages
|
||||||
|
|
||||||
|
if ! systemctl is-active --quiet lxc-net; then
|
||||||
|
systemctl disable --now dnsmasq
|
||||||
|
apt-get install -y -qq lxc
|
||||||
|
systemctl stop lxc-net
|
||||||
|
sed -i -e '/ConditionVirtualization/d' $root/usr/lib/systemd/system/lxc-net.service
|
||||||
|
systemctl daemon-reload
|
||||||
|
cat >> /etc/default/lxc-net <<EOF
|
||||||
|
LXC_ADDR="$prefix.1"
|
||||||
|
LXC_NETMASK="255.255.255.0"
|
||||||
|
LXC_NETWORK="$prefix.0/24"
|
||||||
|
LXC_DHCP_RANGE="$prefix.2,$prefix.254"
|
||||||
|
LXC_DHCP_MAX="253"
|
||||||
|
EOF
|
||||||
|
systemctl start lxc-net
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_install_docker() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
lxc_container_inside $name lxc_install_docker_inside
|
||||||
|
}
|
||||||
|
|
||||||
|
function lxc_install_docker_inside() {
|
||||||
|
lxc_apt_install_inside docker.io docker-compose
|
||||||
|
}
|
136
pkg/runner/lxc-helpers.sh
Executable file
136
pkg/runner/lxc-helpers.sh
Executable file
|
@ -0,0 +1,136 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
source $(dirname $0)/lxc-helpers-lib.sh
|
||||||
|
|
||||||
|
function verbose() {
|
||||||
|
set -x
|
||||||
|
PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
|
||||||
|
LXC_VERBOSE=true
|
||||||
|
}
|
||||||
|
|
||||||
|
function help() {
|
||||||
|
cat <<'EOF'
|
||||||
|
lxc-helpers.sh - LXC container management helpers
|
||||||
|
|
||||||
|
SYNOPSIS
|
||||||
|
|
||||||
|
lxc-helpers.sh [-v|--verbose] [-h|--help]
|
||||||
|
[-o|--os {bookworm|bullseye} (default bookworm)]
|
||||||
|
command [arguments]
|
||||||
|
|
||||||
|
DESCRIPTION
|
||||||
|
|
||||||
|
A thin shell based layer on top of LXC to create, populate, run and
|
||||||
|
destroy LXC containers. A container is created from a copy of an
|
||||||
|
existing container.
|
||||||
|
|
||||||
|
CREATE AND DESTROY
|
||||||
|
|
||||||
|
lxc_prepare_environment
|
||||||
|
|
||||||
|
Install LXC dependencies.
|
||||||
|
|
||||||
|
lxc_container_create `name`
|
||||||
|
|
||||||
|
Create the `name` container.
|
||||||
|
|
||||||
|
lxc_container_mount `name` `path`
|
||||||
|
|
||||||
|
Configure `name` container to bind mount `path` so that it is
|
||||||
|
also accessible at `path` from within the container.
|
||||||
|
|
||||||
|
lxc_container_start `name`
|
||||||
|
|
||||||
|
Start the `name` container.
|
||||||
|
|
||||||
|
lxc_container_stop `name`
|
||||||
|
|
||||||
|
Stop the `name` container.
|
||||||
|
|
||||||
|
lxc_container_destroy `name`
|
||||||
|
|
||||||
|
Call lxc_container_stop `name` and destroy the container.
|
||||||
|
|
||||||
|
lxc_template_release
|
||||||
|
|
||||||
|
Echo the name of the container for the Operating System
|
||||||
|
specified with `--os`.
|
||||||
|
|
||||||
|
lxc_build_template `existing_container` `new_container`
|
||||||
|
|
||||||
|
Copy `existing_container` into `new_container`. If
|
||||||
|
`existing_container` is equal to $(lxc-helpers.sh lxc_template_release) it
|
||||||
|
will be created on demand.
|
||||||
|
|
||||||
|
ACTIONS IN THE CONTAINER
|
||||||
|
|
||||||
|
For some command lxc_something `name` that can be called from outside the container
|
||||||
|
there is an equivalent function lxc_something_inside that can be called from inside
|
||||||
|
the container.
|
||||||
|
|
||||||
|
lxc_install_lxc `name` `prefix`
|
||||||
|
lxc_install_lxc_inside `prefix`
|
||||||
|
|
||||||
|
Install LXC in the `name` container to allow the creation of
|
||||||
|
named containers. `prefix` is a class C IP prefix from which
|
||||||
|
containers will obtain their IP (for instance 10.40.50).
|
||||||
|
|
||||||
|
lxc_container_run `name` command [options...]
|
||||||
|
|
||||||
|
Run the `command` within the `name` container.
|
||||||
|
|
||||||
|
lxc_container_run_script `name` `path`
|
||||||
|
lxc_container_run_script_as `name` `user` `path`
|
||||||
|
|
||||||
|
Run the script found at `path` within the `name` container. The
|
||||||
|
environment is cleared before running the script. The first form
|
||||||
|
will run as root, the second form will impersonate `user`.
|
||||||
|
|
||||||
|
lxc_container_user_install `name` `user_id` `user` [`homedir` default `/home`]
|
||||||
|
|
||||||
|
Create the `user` with `user_id` in the `name` container with a
|
||||||
|
HOME at `/homedir/user`. Passwordless sudo permissions are
|
||||||
|
granted to `user`. It is made a member of the groups docker, kvm
|
||||||
|
and libvirt if they exist already. A SSH key is created.
|
||||||
|
|
||||||
|
Example: lxc_container_user_install mycontainer $(id -u) $USER
|
||||||
|
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
function main() {
|
||||||
|
local options=$(getopt -o hvo --long help,verbose,os: -- "$@")
|
||||||
|
[ $? -eq 0 ] || {
|
||||||
|
echo "Incorrect options provided"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
eval set -- "$options"
|
||||||
|
while true; do
|
||||||
|
case "$1" in
|
||||||
|
-v | --verbose)
|
||||||
|
verbose
|
||||||
|
;;
|
||||||
|
-h | --help)
|
||||||
|
help
|
||||||
|
;;
|
||||||
|
-o | --os)
|
||||||
|
LXC_CONTAINER_RELEASE=$2
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--)
|
||||||
|
shift
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
shift
|
||||||
|
done
|
||||||
|
|
||||||
|
lxc_maybe_sudo
|
||||||
|
|
||||||
|
"$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
|
@ -3,9 +3,11 @@ package runner
|
||||||
import (
|
import (
|
||||||
"archive/tar"
|
"archive/tar"
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
_ "embed"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
@ -16,6 +18,7 @@ import (
|
||||||
"regexp"
|
"regexp"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/opencontainers/selinux/go-selinux"
|
"github.com/opencontainers/selinux/go-selinux"
|
||||||
|
@ -178,6 +181,89 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) {
|
||||||
return binds, mounts
|
return binds, mounts
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//go:embed lxc-helpers-lib.sh
|
||||||
|
var lxcHelpersLib string
|
||||||
|
|
||||||
|
//go:embed lxc-helpers.sh
|
||||||
|
var lxcHelpers string
|
||||||
|
|
||||||
|
var startTemplate = template.Must(template.New("start").Parse(`#!/bin/bash -xe
|
||||||
|
source $(dirname $0)/lxc-helpers-lib.sh
|
||||||
|
|
||||||
|
LXC_CONTAINER_RELEASE="{{.Release}}"
|
||||||
|
|
||||||
|
function template_act() {
|
||||||
|
echo $(lxc_template_release)-act
|
||||||
|
}
|
||||||
|
|
||||||
|
function install_nodejs() {
|
||||||
|
local name="$1"
|
||||||
|
|
||||||
|
local script=/usr/local/bin/lxc-helpers-install-node.sh
|
||||||
|
|
||||||
|
cat > $(lxc_root $name)/$script <<EOF
|
||||||
|
#!/bin/sh -xe
|
||||||
|
# https://github.com/nodesource/distributions#debinstall
|
||||||
|
apt-get install -qq -y curl git
|
||||||
|
curl -fsSL https://deb.nodesource.com/setup_16.x | bash -
|
||||||
|
apt-get install -y nodejs
|
||||||
|
EOF
|
||||||
|
lxc_container_run_script $name $script
|
||||||
|
}
|
||||||
|
|
||||||
|
function build_template_act() {
|
||||||
|
local name="$(template_act)"
|
||||||
|
|
||||||
|
if lxc_exists $name ; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
lxc_build_template $(lxc_template_release) $name
|
||||||
|
lxc_container_start $name
|
||||||
|
install_nodejs $name
|
||||||
|
lxc_container_stop $name
|
||||||
|
}
|
||||||
|
|
||||||
|
lxc_prepare_environment
|
||||||
|
build_template_act
|
||||||
|
lxc_build_template $(template_act) "{{.Name}}"
|
||||||
|
lxc_container_mount "{{.Name}}" "{{ .Root }}"
|
||||||
|
lxc_container_start "{{.Name}}"
|
||||||
|
`))
|
||||||
|
|
||||||
|
var stopTemplate = template.Must(template.New("stop").Parse(`#!/bin/bash -x
|
||||||
|
source $(dirname $0)/lxc-helpers-lib.sh
|
||||||
|
|
||||||
|
lxc_container_destroy "{{.Name}}"
|
||||||
|
`))
|
||||||
|
|
||||||
|
func (rc *RunContext) stopHostEnvironment() common.Executor {
|
||||||
|
return func(ctx context.Context) error {
|
||||||
|
logger := common.Logger(ctx)
|
||||||
|
logger.Debugf("stopHostEnvironment")
|
||||||
|
|
||||||
|
var stopScript bytes.Buffer
|
||||||
|
if err := stopTemplate.Execute(&stopScript, struct {
|
||||||
|
Name string
|
||||||
|
Root string
|
||||||
|
}{
|
||||||
|
Name: rc.JobContainer.GetName(),
|
||||||
|
Root: rc.JobContainer.GetRoot(),
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return common.NewPipelineExecutor(
|
||||||
|
rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
|
||||||
|
Name: "workflow/stop-lxc.sh",
|
||||||
|
Mode: 0755,
|
||||||
|
Body: stopScript.String(),
|
||||||
|
}),
|
||||||
|
rc.JobContainer.Exec([]string{rc.JobContainer.GetActPath() + "/workflow/stop-lxc.sh"}, map[string]string{}, "root", "/tmp"),
|
||||||
|
)(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (rc *RunContext) startHostEnvironment() common.Executor {
|
func (rc *RunContext) startHostEnvironment() common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
logger := common.Logger(ctx)
|
logger := common.Logger(ctx)
|
||||||
|
@ -193,7 +279,8 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
|
||||||
cacheDir := rc.ActionCacheDir()
|
cacheDir := rc.ActionCacheDir()
|
||||||
randBytes := make([]byte, 8)
|
randBytes := make([]byte, 8)
|
||||||
_, _ = rand.Read(randBytes)
|
_, _ = rand.Read(randBytes)
|
||||||
miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes))
|
randName := hex.EncodeToString(randBytes)
|
||||||
|
miscpath := filepath.Join(cacheDir, randName)
|
||||||
actPath := filepath.Join(miscpath, "act")
|
actPath := filepath.Join(miscpath, "act")
|
||||||
if err := os.MkdirAll(actPath, 0o777); err != nil {
|
if err := os.MkdirAll(actPath, 0o777); err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -208,6 +295,8 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
|
||||||
}
|
}
|
||||||
toolCache := filepath.Join(cacheDir, "tool_cache")
|
toolCache := filepath.Join(cacheDir, "tool_cache")
|
||||||
rc.JobContainer = &container.HostEnvironment{
|
rc.JobContainer = &container.HostEnvironment{
|
||||||
|
Name: randName,
|
||||||
|
Root: miscpath,
|
||||||
Path: path,
|
Path: path,
|
||||||
TmpDir: runnerTmp,
|
TmpDir: runnerTmp,
|
||||||
ToolCache: toolCache,
|
ToolCache: toolCache,
|
||||||
|
@ -233,7 +322,44 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var startScript bytes.Buffer
|
||||||
|
if err := startTemplate.Execute(&startScript, struct {
|
||||||
|
Name string
|
||||||
|
Template string
|
||||||
|
Release string
|
||||||
|
Repo string
|
||||||
|
Root string
|
||||||
|
TmpDir string
|
||||||
|
Script string
|
||||||
|
}{
|
||||||
|
Name: rc.JobContainer.GetName(),
|
||||||
|
Template: "debian",
|
||||||
|
Release: "bullseye",
|
||||||
|
Repo: "", // step.Environment["CI_REPO"],
|
||||||
|
Root: rc.JobContainer.GetRoot(),
|
||||||
|
TmpDir: runnerTmp,
|
||||||
|
Script: "", // "commands-" + step.Name,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
return common.NewPipelineExecutor(
|
return common.NewPipelineExecutor(
|
||||||
|
rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
|
||||||
|
Name: "workflow/lxc-helpers-lib.sh",
|
||||||
|
Mode: 0755,
|
||||||
|
Body: lxcHelpersLib,
|
||||||
|
}),
|
||||||
|
rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
|
||||||
|
Name: "workflow/lxc-helpers.sh",
|
||||||
|
Mode: 0755,
|
||||||
|
Body: lxcHelpers,
|
||||||
|
}),
|
||||||
|
rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
|
||||||
|
Name: "workflow/start-lxc.sh",
|
||||||
|
Mode: 0755,
|
||||||
|
Body: startScript.String(),
|
||||||
|
}),
|
||||||
|
rc.JobContainer.Exec([]string{rc.JobContainer.GetActPath() + "/workflow/start-lxc.sh"}, map[string]string{}, "root", "/tmp"),
|
||||||
rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
|
rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{
|
||||||
Name: "workflow/event.json",
|
Name: "workflow/event.json",
|
||||||
Mode: 0o644,
|
Mode: 0o644,
|
||||||
|
@ -346,9 +472,13 @@ func (rc *RunContext) startJobContainer() common.Executor {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lifetime := fmt.Sprint(rc.Config.ContainerMaxLifetime.Round(time.Second).Seconds())
|
||||||
|
if lifetime == "0" {
|
||||||
|
lifetime = "infinity"
|
||||||
|
}
|
||||||
rc.JobContainer = container.NewContainer(&container.NewContainerInput{
|
rc.JobContainer = container.NewContainer(&container.NewContainerInput{
|
||||||
Cmd: nil,
|
Cmd: nil,
|
||||||
Entrypoint: []string{"/bin/sleep", fmt.Sprint(rc.Config.ContainerMaxLifetime.Round(time.Second).Seconds())},
|
Entrypoint: []string{"/bin/sleep", lifetime},
|
||||||
WorkingDir: ext.ToContainerPath(rc.Config.Workdir),
|
WorkingDir: ext.ToContainerPath(rc.Config.Workdir),
|
||||||
Image: image,
|
Image: image,
|
||||||
Username: username,
|
Username: username,
|
||||||
|
@ -546,12 +676,22 @@ func (rc *RunContext) IsHostEnv(ctx context.Context) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RunContext) stopContainer() common.Executor {
|
func (rc *RunContext) stopContainer() common.Executor {
|
||||||
return rc.stopJobContainer()
|
return func(ctx context.Context) error {
|
||||||
|
image := rc.platformImage(ctx)
|
||||||
|
if strings.EqualFold(image, "-self-hosted") {
|
||||||
|
return rc.stopHostEnvironment()(ctx)
|
||||||
|
}
|
||||||
|
return rc.stopJobContainer()(ctx)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rc *RunContext) closeContainer() common.Executor {
|
func (rc *RunContext) closeContainer() common.Executor {
|
||||||
return func(ctx context.Context) error {
|
return func(ctx context.Context) error {
|
||||||
if rc.JobContainer != nil {
|
if rc.JobContainer != nil {
|
||||||
|
image := rc.platformImage(ctx)
|
||||||
|
if strings.EqualFold(image, "-self-hosted") {
|
||||||
|
return rc.stopHostEnvironment()(ctx)
|
||||||
|
}
|
||||||
return rc.JobContainer.Close()(ctx)
|
return rc.JobContainer.Close()(ctx)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -44,10 +44,12 @@ func (sal *stepActionLocal) main() common.Executor {
|
||||||
return func(filename string) (io.Reader, io.Closer, error) {
|
return func(filename string) (io.Reader, io.Closer, error) {
|
||||||
tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, path.Join(cpath, filename))
|
tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, path.Join(cpath, filename))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
common.Logger(context.Background()).Debugf("stepActionLocal reader %s failed %v", path.Join(cpath, filename), err)
|
||||||
return nil, nil, os.ErrNotExist
|
return nil, nil, os.ErrNotExist
|
||||||
}
|
}
|
||||||
treader := tar.NewReader(tars)
|
treader := tar.NewReader(tars)
|
||||||
if _, err := treader.Next(); err != nil {
|
if _, err := treader.Next(); err != nil {
|
||||||
|
common.Logger(context.Background()).Debugf("stepActionLocal reader %s failed %v", path.Join(cpath, filename), err)
|
||||||
return nil, nil, os.ErrNotExist
|
return nil, nil, os.ErrNotExist
|
||||||
}
|
}
|
||||||
return treader, tars, nil
|
return treader, tars, nil
|
||||||
|
|
Loading…
Reference in a new issue