feat: systemd unit example for managing LXC containers (#451)

Refs: https://code.forgejo.org/forgejo/runner/issues/450
Reviewed-on: https://code.forgejo.org/forgejo/runner/pulls/451
Reviewed-by: Michael Kriese <michael.kriese@gmx.de>
Co-authored-by: Earl Warren <contact@earl-warren.org>
Co-committed-by: Earl Warren <contact@earl-warren.org>
pull/457/head
Earl Warren 2025-01-25 12:24:31 +00:00 committed by earl-warren
parent 5970e45df4
commit 5f92ffdbc6
No known key found for this signature in database
GPG Key ID: F128CBE6AB3A7201
4 changed files with 518 additions and 2 deletions

View File

@ -0,0 +1,132 @@
# SPDX-License-Identifier: MIT
on:
push:
branches:
- 'main'
pull_request:
env:
SERIAL: "30"
LIFETIME: "60"
SYSTEMD_OPTIONS: "--no-pager --full"
jobs:
example-lxc-systemd:
if: github.repository_owner != 'forgejo-integration' && github.repository_owner != 'forgejo-experimental' && github.repository_owner != 'forgejo-release'
runs-on: lxc-bookworm
steps:
- uses: https://data.forgejo.org/actions/checkout@v4
- name: forgejo-runner-service.sh dependencies
# run before setup-forgejo because it installs LXC and
# this would do nothing (false positive if a bug sneaks in)
run: |
set -x
cd examples/lxc-systemd
VERBOSE=true ./forgejo-runner-service.sh dependencies
lxc-ls
- id: forgejo
uses: https://data.forgejo.org/actions/setup-forgejo@v2.0.7
with:
user: root
password: admin1234
binary: https://code.forgejo.org/forgejo/forgejo/releases/download/v7.0.12/forgejo-7.0.12-linux-amd64
# must be the same as LXC_IPV4_PREFIX in examples/lxc-systemd/forgejo-runner-service.sh
lxc-ip-prefix: 10.105.7
- name: forgejo-runner-service.sh env
run: |
set -x
# this Forgejo instance needs to be reachable from within the LXC
# container created by forgejo-runner-service.sh
url=http://root:admin1234@${{ steps.forgejo.outputs.host-port }}
docker ps --all
export PATH=$(dirname /tmp/*/forgejocli):$PATH
token=$(su -c 'forgejocli -- actions generate-runner-token' forgejo)
cat > /tmp/env <<EOF
export INPUTS_SERIAL=${{ env.SERIAL }}
export INPUTS_FORGEJO=$url
export INPUTS_TOKEN=$token
export INPUTS_LIFETIME=${{ env.LIFETIME }}
export VERBOSE=true
EOF
- name: forgejo-runner-service.sh create
run: |
eval $(cat /tmp/env)
set -x
./examples/lxc-systemd/forgejo-runner-service.sh install_runner
./examples/lxc-systemd/forgejo-runner-service.sh lxc_create
./examples/lxc-systemd/forgejo-runner-service.sh inside ensure_configuration_and_registration
./examples/lxc-systemd/forgejo-runner-service.sh service_create
- name: forgejo-runner-service.sh assert
run: |
eval $(cat /tmp/env)
set -x
runner=/etc/forgejo-runner/$INPUTS_SERIAL/.runner
cat $runner
test "$(hostname)-${{ env.SERIAL }}" = $(jq -r .name < $runner)
config=/etc/forgejo-runner/$INPUTS_SERIAL/config.yml
cat $config
grep --quiet data.forgejo.org $config
env=/etc/forgejo-runner/$INPUTS_SERIAL/env
cat $env
grep --quiet INPUTS_SERIAL=$INPUTS_SERIAL $env
echo VERBOSE=true >> $env
service=/etc/systemd/system/forgejo-runner@.service
cat $service
- name: forgejo-runner-service.sh start / stop
run: |
set -x
serial=${{ env.SERIAL }}
all="${{ env.SYSTEMD_OPTIONS }}"
systemctl start forgejo-runner@$serial
systemctl $all status forgejo-runner@$serial
started_running=/etc/forgejo-runner/$serial/started-running
killed_gracefully=/etc/forgejo-runner/$serial/killed-gracefully
stopped_gracefully=/etc/forgejo-runner/$serial/stopped-gracefully
retry --delay 5 --times 20 cp -a $started_running /tmp/first-run
retry --delay 1 --times 30 grep --quiet 'Starting runner daemon' /var/log/forgejo-runner/$serial.log
systemctl stop forgejo-runner@$serial
! systemctl $all status forgejo-runner@$serial
ls -l /etc/forgejo-runner/$serial
test -f $killed_gracefully
test -f $stopped_gracefully
systemctl start forgejo-runner@$serial
retry --delay 5 --times 20 cp -a $started_running /tmp/second-run
! test -f $killed_gracefully
! test -f $stopped_gracefully
lifetime=${{ env.LIFETIME }}
# give it time to restart at least once
ls -l /etc/forgejo-runner/$serial
sleep $lifetime ; sleep $lifetime
ls -l /etc/forgejo-runner/$serial
! test -f $killed_gracefully
! test -f $stopped_gracefully
retry --delay 5 --times 20 cp -a $started_running /tmp/third-run
systemctl stop forgejo-runner@$serial
ls -l /etc/forgejo-runner/$serial
test -f $killed_gracefully
test -f $stopped_gracefully
ls -l /tmp/*-run
test /tmp/first-run -ot /tmp/second-run
test /tmp/second-run -ot /tmp/third-run
- name: forgejo-runner-service.sh status & destroy
if: always()
run: |
eval $(cat /tmp/env)
set -x
cat /var/log/forgejo-runner/${{ env.SERIAL }}.log || true
journalctl ${{ env.SYSTEMD_OPTIONS }} --unit forgejo-runner@${{ env.SERIAL }} || true
./examples/lxc-systemd/forgejo-runner-service.sh lxc_destroy

View File

@ -1,10 +1,10 @@
This directory contains a collection of usage and deployment examples.
Workflow examples can be found [in the documentation](https://forgejo.org/docs/next/user/actions/)
and in the [sources of the setup-forgejo](https://code.forgejo.org/actions/setup-forgejo/src/branch/main/testdata) action.
Workflow examples can be found [in the documentation](https://forgejo.org/docs/next/user/actions/).
| Section | Description |
|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| [`docker`](docker) | using the host docker server by mounting the socket |
| [`LXC systemd`](lxc-systemd) | systemd unit managing LXC containers dedicated to a single runner |
| [`docker-compose`](docker-compose) | all in one docker-compose with the Forgejo server, the runner and docker in docker |
| [`kubernetes`](kubernetes) | a sample deployment for the Forgejo runner |

View File

@ -0,0 +1,66 @@
forgejo-runner-service.sh installs a [Forgejo runner](https://forgejo.org/docs/next/admin/runner-installation/) within an [LXC container](https://linuxcontainers.org/lxc/) and runs it from a systemd service.
## Quickstart
- Install: `sudo wget -O /usr/local/bin/forgejo-runner-service.sh https://code.forgejo.org/forgejo/runner/raw/branch/main/examples/lxc-systemd/forgejo-runner-service.sh && sudo chmod +x /usr/local/bin/forgejo-runner-service.sh`
- Obtain a runner registration token ($TOKEN)
- Choose a serial number that is not already in use in `/etc/forgejo-runner`
- Create a runner `INPUTS_SERIAL=30 INPUTS_TOKEN=$TOKEN INPUTS_FORGEJO=https://code.forgejo.org forgejo-runner-service.sh`
- Start `systemctl enable --now forgejo-runner@$INPUTS_SERIAL`
- Monitor with:
- `systemctl status forgejo-runner@$INPUTS_SERIAL`
- `tail --follow=name /var/log/forgejo-runner/$INPUTS_SERIAL.log`
## Installation or upgrade
- `sudo wget -O /usr/local/bin/forgejo-runner-service.sh https://code.forgejo.org/forgejo/runner/raw/branch/main/examples/lxc-systemd/forgejo-runner-service.sh && sudo chmod +x /usr/local/bin/forgejo-runner-service.sh`
## Description
- Each runner is assigned a unique serial number (`$INPUTS_SERIAL`)
- The configuration is in `/etc/forgejo-runner/$INPUTS_SERIAL`
- The environment variables are in `/etc/forgejo-runner/$INPUTS_SERIAL/env`
- The cache is in `/var/lib/forgejo-runner/runner-$INPUTS_SERIAL`
- The systemd service unit is `forgejo-runner@$INPUTS_SERIAL`
- The logs of the runner daemon are in `/var/log/forgejo-runner/$INPUTS_SERIAL.log`
## How it works
- Creating a runner (for instance with `INPUTS_SERIAL=30 INPUTS_TOKEN=$TOKEN INPUTS_FORGEJO=https://code.forgejo.org forgejo-runner-service.sh`) will:
- use `$INPUTS_TOKEN` to register on `$INPUTS_FORGEJO` and save the result in the `/etc/forgejo-runner/$INPUTS_SERIAL/.runner` file
- generate a default configuration file in the `/etc/forgejo-runner/$INPUTS_SERIAL/config.yml` file which can then be manually edited
- Each runner is launched in a dedicated LXC container named `runner-$INPUTS_SERIAL-lxc` with the following bind mounts:
- `/etc/forgejo-runner/$INPUTS_SERIAL`
- `/var/lib/forgejo-runner/runner-$INPUTS_SERIAL/.cache/actcache`
- `systemctl start forgejo-runner@$INPUTS_SERIAL` will do the following when it starts and every `$INPUTS_LIFETIME` interval after that:
- attempt to gracefully stop (SIGTERM) the runner, waiting for all jobs to complete
- forcibly kill the runner if it does not stop within 6h
- shutdown the LXC container and delete it (the volumes bind mounted are preserved)
- create a brand new LXC container (with the specified `$INPUTS_LXC_CONFIG`)
- install and run a Forgejo runner daemon in the LXC container using `/etc/forgejo-runner/$INPUTS_SERIAL/config.yml`
- redirect the output of the runner to `/var/log/forgejo-runner/$INPUTS_SERIAL.log`
- `systemctl stop forgejo-runner@$INPUTS_SERIAL` will stop the runner but keep the LXC container running
## Creation
The creation of a new runner is driven by the following environment variables:
- `INPUTS_SERIAL`: unique number in the range `[10-100]` (check `/etc/forgejo-runner`)
- `INPUTS_TOKEN`: a runner registration token obtained from the web UI
- `INPUTS_FORGEJO`: the Forgejo instance from which `INPUTS_TOKEN` was obtained (e.g. https://code.forgejo.org)
- `INPUTS_RUNNER_VERSION`: the version of the Forgejo runner as found in https://code.forgejo.org/forgejo/runner/releases (e.g. 6.2.0)
- `INPUTS_LXC_CONFIG`: the value of the `--config` argument of [lxc-helpers](https://code.forgejo.org/forgejo/lxc-helpers/#usage) used when creating the LXC container for the runner (e.g. `docker`)
- `INPUTS_LIFETIME`: the LXC container is re-created when its lifetime expires (e.g. 7d)
## Hacking
- An existing LXC configuration will not be modified. If `lxc-ls` exists, it is assumed that LXC is configured and ready to be used.
- Migrating an existing runner:
```sh
serial=10
mkdir /etc/forgejo-runner/$serial
cp .runner config.yml /etc/forgejo-runner/$serial
INPUTS_SERIAL=$serial INPUTS_FORGEJO=https://code.forgejo.org forgejo-runner-service.sh
systemctl status forgejo-runner@$serial
```
- Set debug by adding `VERBOSE=true` in `/etc/forgejo-runner/$INPUTS_SERIAL/env`

View File

@ -0,0 +1,318 @@
#!/bin/bash
# Copyright Forgejo Authors.
# SPDX-License-Identifier: MIT
set -o pipefail
: ${TMPDIR:=$(mktemp -d)}
export -n TMPDIR
if ! test -d "$TMPDIR"; then
echo "TMPDIR=$TMPDIR is expected to be a directory"
exit 1
fi
trap "rm -fr $TMPDIR" EXIT
: ${INPUTS_LXC_CONFIG:=docker libvirt lxc}
: ${INPUTS_SERIAL:=}
: ${INPUTS_TOKEN:=}
: ${INPUTS_FORGEJO:=https://code.forgejo.org}
: ${INPUTS_LIFETIME:=7d}
: ${INPUTS_RUNNER_VERSION:=6.2.0}
: ${KILL_AFTER:=21600} # 6h == 21600
NODEJS_VERSION=20
DEBIAN_RELEASE=bookworm
YQ_VERSION=v4.45.1
SELF=${BASH_SOURCE[0]}
SELF_FILENAME=$(basename "$SELF")
ETC=/etc/forgejo-runner
LIB=/var/lib/forgejo-runner
LOG=/var/log/forgejo-runner
: ${HOST:=$(hostname)}
LXC_IPV4_PREFIX="10.105.7"
LXC_IPV6_PREFIX="fd91"
LXC_USER_NAME=debian
LXC_USER_ID=1000
if ${VERBOSE:-false}; then
set -ex
PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
# export LXC_VERBOSE=true # use with caution, it will block .forgejo/workflows/example-lxc-systemd.yml
else
set -e
fi
if test $(id -u) != 0; then
SUDO=sudo
fi
function config_inotify() {
if grep --quiet fs.inotify.max_user_instances=8192 /etc/sysctl.conf; then
return
fi
echo fs.inotify.max_user_instances=8192 | $SUDO tee -a /etc/sysctl.conf
$SUDO sysctl -p
}
function dependencies() {
if ! which curl jq retry >/dev/null; then
export DEBIAN_FRONTEND=noninteractive
$SUDO apt-get update -qq
$SUDO apt-get install -y -qq curl jq retry
fi
if ! which lxc-helpers.sh >/dev/null; then
$SUDO curl --fail -sS -o /usr/local/bin/lxc-helpers-lib.sh https://code.forgejo.org/forgejo/lxc-helpers/raw/branch/main/lxc-helpers-lib.sh
$SUDO curl --fail -sS -o /usr/local/bin/lxc-helpers.sh https://code.forgejo.org/forgejo/lxc-helpers/raw/branch/main/lxc-helpers.sh
$SUDO chmod +x /usr/local/bin/lxc-helpers*.sh
fi
if ! which lxc-ls >/dev/null; then
$SUDO lxc-helpers.sh lxc_install_lxc_inside $LXC_IPV4_PREFIX $LXC_IPV6_PREFIX
fi
if ! which yq >/dev/null; then
$SUDO curl -L --fail -sS -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_arm64
$SUDO chmod +x /usr/local/bin/yq
fi
if ! cmp $SELF /usr/local/bin/$SELF_FILENAME; then
cp -a $SELF /usr/local/bin/$SELF_FILENAME
fi
}
function lxc_name() {
echo runner-${INPUTS_SERIAL}-lxc
}
function lxc_destroy() {
$SUDO lxc-destroy -f $(lxc_name) >/dev/null || true
}
function lxc_create() {
local name=$(lxc_name)
local lib=$LIB/$name
local etc=$ETC/$INPUTS_SERIAL
lxc-helpers.sh --config "$INPUTS_LXC_CONFIG" lxc_container_create $name
echo "lxc.start.auto = 1" | sudo tee -a /var/lib/lxc/$name/config
local bin=/var/lib/lxc/$name/rootfs/usr/local/bin
$SUDO cp -a $SELF $bin/$SELF_FILENAME
$SUDO cp -a /usr/local/bin/forgejo-runner-$INPUTS_RUNNER_VERSION $bin/forgejo-runner
$SUDO cp -a /usr/local/bin/yq $bin/yq
$SUDO cp -a $(which jq) $bin/jq
$SUDO mkdir -p $lib/.cache/actcache
$SUDO chown -R $LXC_USER_ID $lib
lxc-helpers.sh lxc_container_mount $name $lib/.cache/actcache
$SUDO mkdir -p $etc
$SUDO chown -R $LXC_USER_ID $etc
lxc-helpers.sh lxc_container_mount $name $etc
lxc-helpers.sh lxc_container_start $name
if echo $INPUTS_LXC_CONFIG | grep --quiet 'docker'; then
lxc-helpers.sh lxc_install_docker $name
fi
if echo $INPUTS_LXC_CONFIG | grep --quiet 'lxc'; then
local ipv4="10.48.$INPUTS_SERIAL"
local ipv6="fd$INPUTS_SERIAL"
lxc-helpers.sh lxc_install_lxc $name $ipv4 $ipv6
fi
lxc-helpers.sh lxc_container_user_install $name $LXC_USER_ID $LXC_USER_NAME
}
function service_create() {
cat >$TMPDIR/forgejo-runner@.service <<EOF
[Unit]
Description=Forgejo runner %i
After=syslog.target
After=network.target
[Service]
Restart=on-success
ExecStart=/usr/local/bin/${SELF_FILENAME} run_in_copy start
ExecStop=/usr/local/bin/${SELF_FILENAME} stop
EnvironmentFile=/etc/forgejo-runner/%i/env
[Install]
WantedBy=multi-user.target
EOF
local service=/etc/systemd/system/forgejo-runner@.service
if test -f $service && cmp $TMPDIR/forgejo-runner@.service $service; then
return
fi
$SUDO mkdir -p $ETC
$SUDO chown -R $LXC_USER_ID $ETC
$SUDO mkdir -p $LOG
$SUDO chown -R $LXC_USER_ID $LOG
$SUDO cp $TMPDIR/forgejo-runner@.service $service
$SUDO systemctl daemon-reload
}
function inside() {
local name=$(lxc_name)
lxc-helpers.sh lxc_container_run $name -- sudo --user $LXC_USER_NAME \
INPUTS_SERIAL=$INPUTS_SERIAL \
INPUTS_TOKEN=$INPUTS_TOKEN \
INPUTS_FORGEJO=$INPUTS_FORGEJO \
INPUTS_LIFETIME=$INPUTS_LIFETIME \
KILL_AFTER=$KILL_AFTER \
VERBOSE=$VERBOSE \
HOST=$HOST \
$SELF_FILENAME "$@"
}
function install_runner() {
local runner=/usr/local/bin/forgejo-runner-$INPUTS_RUNNER_VERSION
if test -f $runner; then
return
fi
$SUDO curl --fail -sS -o $runner https://code.forgejo.org/forgejo/runner/releases/download/v$INPUTS_RUNNER_VERSION/forgejo-runner-$INPUTS_RUNNER_VERSION-linux-amd64
$SUDO chmod +x $runner
}
function ensure_configuration() {
if test -z "$INPUTS_SERIAL"; then
echo "the INPUTS_SERIAL environment variable is not set"
return 1
fi
local etc=$ETC/$INPUTS_SERIAL
$SUDO mkdir -p $etc
if test -f $etc/config; then
INPUTS_LXC_CONFIG=$(cat $etc/config)
else
echo $INPUTS_LXC_CONFIG >$etc/config
fi
$SUDO mkdir -p $LIB/$(lxc_name)/.cache/actcache
}
function ensure_configuration_and_registration() {
local etc=$ETC/$INPUTS_SERIAL
if ! test -f $etc/config.yml; then
forgejo-runner generate-config >$etc/config.yml
cat >$TMPDIR/edit-config <<EOF
.runner.labels = ["docker:docker://data.forgejo.org/oci/node:${NODEJS_VERSION}-${DEBIAN_RELEASE}","lxc:lxc://debian:${DEBIAN_RELEASE}"]
EOF
yq --inplace --from-file $TMPDIR/edit-config $etc/config.yml
fi
if ! test -f $etc/env; then
cat >$etc/env <<EOF
INPUTS_LXC_CONFIG=$INPUTS_LXC_CONFIG
INPUTS_SERIAL=$INPUTS_SERIAL
INPUTS_LIFETIME=$INPUTS_LIFETIME
INPUTS_FORGEJO=$INPUTS_FORGEJO
EOF
fi
if test -f $etc/.runner; then
return
fi
if test -z "$INPUTS_TOKEN"; then
echo "the INPUTS_TOKEN environment variable is not set"
return 1
fi
(
cd $etc
forgejo-runner register --config config.yml --no-interactive \
--token "$INPUTS_TOKEN" \
--name "$HOST-$INPUTS_SERIAL" \
--instance $INPUTS_FORGEJO
)
}
function daemon() {
cd $ETC/$INPUTS_SERIAL
rm -f stopped-* killed-*
touch started-running
set +e
timeout --signal=SIGINT --kill-after=$KILL_AFTER $INPUTS_LIFETIME forgejo-runner --config config.yml daemon
case $? in
0) touch stopped-gracefully ;;
124) touch stopped-timeout ;;
137) touch stopped-forcefully ;;
esac
set -e
}
function start() {
stop
lxc-helpers.sh lxc_container_destroy $(lxc_name)
lxc_create
local log=$LOG/$INPUTS_SERIAL.log
if test -f $log; then
mv $log $log.backup
fi
inside daemon >&$log
}
function kill_runner() {
cd $ETC/$INPUTS_SERIAL
rm -f killed-* started-running
set +e
pkill --exact forgejo-runner
if test $? = 1; then
touch killed-already
return
fi
timeout $KILL_AFTER pidwait --exact forgejo-runner
status=$?
set -e
# pidwait will exit 1 if the process is already gone
# pidwait will exit 0 if the process terminated gracefully before the timeout
if test $status = 0 || test $status = 1; then
touch killed-gracefully
echo "forgejo-runner stopped gracefully"
else
pkill --exact --signal=KILL forgejo-runner
touch killed-forcefully
echo "forgejo-runner stopped forcefully"
fi
}
function stop() {
inside kill_runner
}
function main() {
config_inotify
dependencies
install_runner
service_create
lxc_create
inside ensure_configuration_and_registration
}
#
# ensure an update of the current script does not break a long
# running function (such as `start`) by running from a copy instead
# of the script itself
#
function run_in_copy() {
if test "$#" = 0; then
echo "run_in_copy needs an argument"
return 1
fi
export TMPDIR # otherwise it will not be removed by trap
cp $SELF $TMPDIR/$SELF_FILENAME
exec $TMPDIR/$SELF_FILENAME "$@"
}
"${@:-main}"