Skip to content

Commit d3c7350

Browse files
author
Lennart Weller
committed
Provide a scalable CoreOS example
 - Also includes a container based "QEMU Guest Agent" setup
1 parent 5417057 commit d3c7350

File tree

5 files changed

+215
-0
lines changed

5 files changed

+215
-0
lines changed

‎examples/coreos/README.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# CoreOS multi-machine example setup
2+
3+
### Requirements
4+
* Linux Kernel ~> 4.14
5+
* Libvirt ~> 3.0
6+
* QEMU ~> 2.6
7+
8+
This is a relatively simple scalable example using CoreOS as operating system.
9+
By modifying the `hosts` variable you can kickstart any number of virtual machines
10+
with their own ignition configuration
11+
12+
13+
### Using the QEMU Guest Agent
14+
15+
In case you don't use the networks provided by libvirt you may run into the issue that you won't be able to receive the IP addresses from the VM you create.
16+
17+
Using the QEMU guest agent allows libvirt to pick up the address by hooking itself into the guest operating system.
18+
As CoreOS comes without any guest agents we need to supply it from somewhere.
19+
If the machine has internet access you can edit the `qemu-agent.service` file and remove the `ExecStartPre` line and the docker daemon should download the appropriate container when you activate the service file in the ignition config. If the machine has no access to the internet we need to upload the container from the KVM host.[1]
20+
```bash
21+
$ docker pull docker.io/rancher/os-qemuguestagent:v2.8.1-2
22+
$ docker save docker.io/rancher/os-qemuguestagent:v2.8.1-2 -o /srv/images/qemu-guest-agent.tar
23+
```
24+
25+
Make sure the relevant blocks are uncommented in the domain definition and the ignition config. The ignition configuration should include the two additional files `docker-images.mount` and `qemu-agent.service`. Note that the`qemu-guest-agent.tar` needs to be local to the KVM host and not the machine running terraform.
26+
27+
28+
### Known Bugs
29+
1. Before Linux 4.14-rc2 the graphics option "autoport" will not work and libvirt will try to create all machines with the same Spice/VNC port
30+
2. Below libvirt v3 the generated ignition id will change when the number of machines is changed causing a destroy/create for all machines.
31+
32+
33+
[1]: Based on the work of [@tommyknows](https://github.com/dmacvicar/terraform-provider-libvirt/issues/364#issuecomment-442164364) and [@remoe](https://github.com/dmacvicar/terraform-provider-libvirt/issues/364#issuecomment-443456552).

‎examples/coreos/ignition.tf‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Terraform ignition configuration
2+
# All configuration options are detailed at
3+
# https://www.terraform.io/docs/providers/ignition/index.html
4+
5+
data "ignition_config" "startup" {
6+
users = [
7+
"${data.ignition_user.core.id}",
8+
]
9+
10+
files = [
11+
"${element(data.ignition_file.hostname.*.id, count.index)}",
12+
]
13+
14+
## Relevant for the QEMU Guest Agent example
15+
#systemd = [
16+
# "${data.ignition_systemd_unit.mount-images.id}",
17+
# "${data.ignition_systemd_unit.qemu-agent.id}"
18+
#]
19+
count = "${var.hosts}"
20+
}
21+
22+
# Replace the default hostname with our generated one
23+
data "ignition_file" "hostname" {
24+
filesystem = "root" # default `ROOT` filesystem
25+
path = "/etc/hostname"
26+
mode = 420 # decimal 0644
27+
28+
content {
29+
content = "${format(var.hostname_format, count.index + 1)}"
30+
}
31+
32+
count = "${var.hosts}"
33+
}
34+
35+
# Example configuration for the basic `core` user
36+
data "ignition_user" "core" {
37+
name = "core"
38+
39+
#Example password: foobar
40+
password_hash = "$5$XMoeOXG6$8WZoUCLhh8L/KYhsJN2pIRb3asZ2Xos3rJla.FA1TI7"
41+
42+
# Preferably use the ssh key auth instead
43+
#ssh_authorized_keys = "${list()}"
44+
}
45+
46+
## Relevant for the QEMU Guest Agent example
47+
#data "ignition_systemd_unit" "mount-images" {
48+
# name = "mnt-images.mount"
49+
# enabled = true
50+
# content = "${file("${path.module}/qemu-agent/docker-images.mount")}"
51+
#}
52+
53+
54+
## Relevant for the QEMU Guest Agent example
55+
#data "ignition_systemd_unit" "qemu-agent" {
56+
# name = "qemu-agent.service"
57+
# enabled = true
58+
# content = "${file("${path.module}/qemu-agent/qemu-agent.service")}"
59+
#}
60+

‎examples/coreos/main.tf‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# -[Provider]--------------------------------------------------------------
2+
provider "libvirt" {
3+
uri = "qemu:///system"
4+
}
5+
6+
# -[Variables]-------------------------------------------------------------
7+
variable "hosts" {
8+
default = 1
9+
}
10+
11+
variable "hostname_format" {
12+
type = "string"
13+
default = "coreos%02d"
14+
}
15+
16+
variable "libvirt_provider" {
17+
type = "string"
18+
}
19+
20+
# -[Resources]-------------------------------------------------------------
21+
resource "libvirt_volume" "coreos-disk" {
22+
name = "${format(var.hostname_format, count.index + 1)}.qcow2"
23+
count = "${var.hosts}"
24+
base_volume_name = "coreos_production_qemu"
25+
pool = "default"
26+
format = "qcow2"
27+
}
28+
29+
# Loading ignition configs in QEMU requires at least QEMU v2.6
30+
resource "libvirt_ignition" "ignition" {
31+
name = "${format(var.hostname_format, count.index + 1)}-ignition"
32+
pool = "default"
33+
count = "${var.hosts}"
34+
content = "${element(data.ignition_config.startup.*.rendered, count.index)}"
35+
}
36+
37+
# Create the virtual machines
38+
resource "libvirt_domain" "coreos-machine" {
39+
count = "${var.hosts}"
40+
name = "${format(var.hostname_format, count.index + 1)}"
41+
vcpu = "1"
42+
memory = "2048"
43+
44+
## Use qemu-agent in conjunction with the container
45+
#qemu_agent = true
46+
coreos_ignition = "${element(libvirt_ignition.ignition.*.id, count.index)}"
47+
48+
disk {
49+
volume_id = "${element(libvirt_volume.coreos-disk.*.id, count.index)}"
50+
}
51+
52+
graphics {
53+
## Bug in linux up to 4.14-rc2
54+
## https://bugzilla.redhat.com/show_bug.cgi?id=1432684
55+
## No Spice/VNC available if more then one machine is generated at a time
56+
## Comment the address line, uncomment the none line and the console block below
57+
#listen_type = "none"
58+
listen_type = "address"
59+
}
60+
61+
## Makes the tty0 available via `virsh console`
62+
#console {
63+
# type = "pty"
64+
# target_port = "0"
65+
#}
66+
67+
network_interface {
68+
network_name = "default"
69+
70+
# Requires qemu-agent container if network is not native to libvirt
71+
wait_for_lease = true
72+
}
73+
74+
## mounts filesystem local to the kvm host. used to patch in the
75+
## qemu-guest-agent as docker container
76+
#filesystem {
77+
# source = "/srv/images/"
78+
# target = "qemu_docker_images"
79+
# readonly = true
80+
#}
81+
}
82+
83+
# -[Output]-------------------------------------------------------------
84+
output "ipv4" {
85+
value = "${libvirt_domain.coreos-machine.*.network_interface.0.addresses}"
86+
}

‎examples/coreos/qemu-agent/docker-images.mount‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[Unit]
2+
Before=local-fs.target
3+
[Mount]
4+
What=qemu_docker_images
5+
Where=/mnt/images
6+
Options=ro,trans=virtio,version=9p2000.L
7+
Type=9p
8+
[Install]
9+
WantedBy=local-fs.target

‎examples/coreos/qemu-agent/qemu-agent.service‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
[Unit]
2+
Description=QEMU Agent
3+
After=docker.service
4+
[Service]
5+
ExecStartPre=/usr/bin/docker load -i /mnt/images/qemu-guest-agent.tar
6+
ExecStart=/usr/bin/docker run \
7+
--privileged=true \
8+
--cap-add=ALL \
9+
--net=host \
10+
--name=qemu-guest-agent \
11+
-e container=1 \
12+
-e HOST=/host \
13+
-e TERM=xterm \
14+
-v /dev/virtio-ports:/dev/virtio-ports \
15+
-v /etc/os-release:/etc/os-release:ro \
16+
-v /dev:/dev \
17+
-v /proc:/hostproc \
18+
-v /run/systemd:/run/systemd \
19+
-v /var/log/qemu-ga:/var/log/qemu-ga:rw \
20+
-v /var/run/docker.sock:/var/run/docker.sock:ro \
21+
--entrypoint qemu-ga \
22+
rancher/os-qemuguestagent:v2.8.1-2 \
23+
-- -v
24+
Restart=on-failure
25+
RestartSec=10
26+
[Install]
27+
WantedBy=multi-user.target

0 commit comments

Comments
 (0)