diff options
Diffstat (limited to 'default/steps')
22 files changed, 1817 insertions, 0 deletions
diff --git a/default/steps/aliases/defaults.yaml b/default/steps/aliases/defaults.yaml new file mode 100644 index 0000000..6cf723b --- /dev/null +++ b/default/steps/aliases/defaults.yaml @@ -0,0 +1,169 @@ +write_local: + - exec_local: | + mkdir -p $(dirname @1); + cat >@1 <<EOF_KAMELEON_INTERNAL + @2 + EOF_KAMELEON_INTERNAL + +write_in: + - exec_in: | + mkdir -p $(dirname @1); + cat >@1 <<EOF_KAMELEON_INTERNAL + @2 + EOF_KAMELEON_INTERNAL + +write_out: + - exec_out: | + mkdir -p $(dirname @1); + cat >@1 <<EOF_KAMELEON_INTERNAL + @2 + EOF_KAMELEON_INTERNAL + +append_local: + - exec_local: | + mkdir -p $(dirname @1); + cat >>@1 <<EOF_KAMELEON_INTERNAL + @2 + EOF_KAMELEON_INTERNAL + +append_in: + - exec_in: | + mkdir -p $(dirname @1); + cat >>@1 <<EOF_KAMELEON_INTERNAL + @2 + EOF_KAMELEON_INTERNAL + +append_out: + - exec_out: | + mkdir -p $(dirname @1); + cat >>@1 <<EOF_KAMELEON_INTERNAL + @2 + EOF_KAMELEON_INTERNAL + +write_raw_local: + - exec_local: | + mkdir -p $(dirname @1); + cat >@1 <<'EOF_KAMELEON_INTERNAL' + @2 + EOF_KAMELEON_INTERNAL + +write_raw_in: + - exec_in: | + mkdir -p $(dirname @1); + cat >@1 <<'EOF_KAMELEON_INTERNAL' + @2 + EOF_KAMELEON_INTERNAL + +write_raw_out: + - exec_out: | + mkdir -p $(dirname @1); + cat >@1 <<'EOF_KAMELEON_INTERNAL' + @2 + EOF_KAMELEON_INTERNAL + +append_raw_local: + - exec_local: | + mkdir -p $(dirname @1); + cat >>@1 <<'EOF_KAMELEON_INTERNAL' + @2 + EOF_KAMELEON_INTERNAL + +append_raw_in: + - exec_in: | + mkdir -p $(dirname @1); + cat >>@1 <<'EOF_KAMELEON_INTERNAL' + @2 + EOF_KAMELEON_INTERNAL + +append_raw_out: + - exec_out: | + mkdir -p $(dirname @1); + cat >>@1 <<'EOF_KAMELEON_INTERNAL' + @2 + EOF_KAMELEON_INTERNAL + +local2out: + - exec_out: | + mkdir -p $(dirname @2) + - pipe: + - exec_local: cat @1 + - exec_out: cat > @2 + +local2in: + - exec_in: mkdir -p $(dirname @2) + - pipe: + - exec_local: cat @1 + - exec_in: cat > @2 + +out2local: + - exec_local: mkdir -p $(dirname @2) + - pipe: + - exec_out: cat @1 + - exec_local: cat > @2 + +out2in: + - exec_in: mkdir -p $(dirname @2) + - pipe: + - exec_out: cat @1 + - exec_in: cat > @2 + +in2local: + - exec_local: mkdir -p $(dirname @2) + - pipe: + - exec_in: cat @1 + - exec_local: cat > @2 + +in2out: + - exec_out: mkdir -p $(dirname @2) + - pipe: + - exec_in: cat @1 + - exec_out: cat > @2 + +check_cmd_out: + - rescue: + - exec_out: command -V @1 2> /dev/null + - breakpoint: "@1 is missing from out_context" + +check_cmd_local: + - on_bootstrap_init: + - rescue: + - exec_local: command -V @1 2> /dev/null + - breakpoint: "@1 is missing from local_context" + +check_cmd_in: + - rescue: + - exec_in: command -V @1 2> /dev/null + - breakpoint: "@1 is missing from in_context" + +umount_out: + - exec_out: | + echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true + +umount_local: + - exec_local: | + echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true + +umount_in: + - exec_in: | + echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true + +download_file_in: + - exec_in: __download "@1" "@2" + +download_file_out: + - exec_out: __download "@1" "@2" + +download_file_local: + - exec_local: __download "@1" "@2" + +download_recipe_build_local: + - exec_local: __download_recipe_build "@1" "@2" "@3" "@4" "@5" "@6" "@7" + +download_kadeploy_environment_image_local: + - exec_local: __download_kadeploy_environment_image "@1" "@2" "@3" "@4" "@5" + +apt-get_in: + - exec_in: DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes @1 2>&1 + +apt-get_out: + - exec_out: DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes @1 2>&1 diff --git a/default/steps/bootstrap/debian/prepare_autoinstall.yaml b/default/steps/bootstrap/debian/prepare_autoinstall.yaml new file mode 100644 index 0000000..f737d20 --- /dev/null +++ b/default/steps/bootstrap/debian/prepare_autoinstall.yaml @@ -0,0 +1,11 @@ +- copy_autoinstall_script_to_http_directory: + - exec_local: mkdir -p $${http_directory} + - exec_local: cp $${base_preseed_path} $${http_directory}/preseed.cfg + +- customize_preseed: + - exec_local: sed -i -e 's|\(d-i passwd/root-password password \).*|\1$${root_password}|g' $${http_directory}/preseed.cfg + - exec_local: sed -i -e 's|\(d-i passwd/root-password-again password \).*|\1$${root_password}|g' $${http_directory}/preseed.cfg + - exec_local: sed -i -e 's|\(d-i mirror/http/hostname string \).*|\1$${deb_mirror_hostname}|g' $${http_directory}/preseed.cfg + - exec_local: sed -i -e 's|\(d-i mirror/http/directory string \).*|\1$${deb_mirror_directory}|g' $${http_directory}/preseed.cfg + - exec_local: sed -i -e 's|\(d-i apt-setup/security_host string \).*|\1$${deb_security_hostname}|g' $${http_directory}/preseed.cfg + - exec_local: sed -i -e 's|\(d-i apt-setup/security_path string \).*|\1$${deb_security_directory}|g' $${http_directory}/preseed.cfg diff --git a/default/steps/bootstrap/download_installer.yaml b/default/steps/bootstrap/download_installer.yaml new file mode 100644 index 0000000..f15f58c --- /dev/null +++ b/default/steps/bootstrap/download_installer.yaml @@ -0,0 +1,31 @@ +- download_installer: + - test: + - exec_local: test -n "$${installer_iso_url}" -o -n "$${installer_iso_finder_helper}" + - group: + - test: + - exec_local: test -z "$${installer_iso_url}" + - exec_local: | + echo "Looking for the netinstall iso URL for $${installer_iso_finder_args}" + DOWNLOAD_SRC_URL=$(python2 $${installer_iso_finder_helper} $${installer_iso_finder_args}) + - download_file_local: + - $${installer_iso_url} + - $${qemu_iso_path} + - exec_local: unset DOWNLOAD_SRC_URL + - group: + - test: + - exec_local: test -n "$${installer_kernel_url}" + - download_file_local: + - $${installer_kernel_url} + - $${qemu_kernel_path} + - test: + - exec_local: test -n "$${installer_initrd_url}" + - download_file_local: + - $${installer_initrd_url} + - $${qemu_initrd_path} + +- delete_installer: + - on_checkpoint: skip + - on_export_clean: + - exec_local: rm -f $${qemu_iso_path} + - exec_local: rm -f $${qemu_kernel_path} + - exec_local: rm -f $${qemu_initrd_path} diff --git a/default/steps/bootstrap/prepare_appliance.yaml b/default/steps/bootstrap/prepare_appliance.yaml new file mode 100644 index 0000000..4f597c4 --- /dev/null +++ b/default/steps/bootstrap/prepare_appliance.yaml @@ -0,0 +1,33 @@ +- insecure_ssh_key: $${kameleon_cwd}/insecure_ssh_key + +- generate_ssh_keys: + - check_cmd_local: ssh-keygen + - exec_local: echo -e 'y\n' | ssh-keygen -q -t rsa -b 4096 -f $${insecure_ssh_key} -N '' + - exec_local: cat $${insecure_ssh_key} + +- inject_ssh_private_key: + - check_cmd_local: virt-customize + - exec_local: | + virt-customize \ + -a $${image_disk}.$${image_format} \ + --run-command 'mkdir -p /root/.ssh' \ + --upload $${insecure_ssh_key}.pub:/root/.ssh/.kameleon_authorized_keys \ + --run-command 'touch /root/.ssh/authorized_keys' \ + --run-command 'cp /root/.ssh/authorized_keys /root/.ssh/authorized_keys.bak' \ + --run-command 'cat /root/.ssh/.kameleon_authorized_keys >> /root/.ssh/authorized_keys' \ + --run-command 'chmod 700 /root/.ssh' \ + --run-command 'chmod -R go-rw /root/.ssh' \ + --run-command 'chown -R root:root /root/.ssh' + - on_export_init: + - exec_local: | + virt-customize \ + -a $${image_disk}.$${image_format} \ + --run-command 'mv /root/.ssh/authorized_keys.bak /root/.ssh/authorized_keys' \ + --delete /root/.ssh/.kameleon_authorized_keys + +- add_insecure_key_to_ssh_config: + - on_checkpoint: redo + - exec_local: | + cat <<EOF >> $${ssh_config_file} + IdentityFile $${insecure_ssh_key} + EOF diff --git a/default/steps/bootstrap/prepare_disk.yaml b/default/steps/bootstrap/prepare_disk.yaml new file mode 100644 index 0000000..9c3dce4 --- /dev/null +++ b/default/steps/bootstrap/prepare_disk.yaml @@ -0,0 +1,10 @@ +- create_initial_image: + - check_cmd_local: qemu-img + - exec_local: | + rm -f $${image_disk}.$${image_format} + qemu-img create -f qcow2 $${image_disk}.$${image_format} $${qemu_image_size} + +- delete_initial_image: + - on_checkpoint: skip + - on_export_clean: + - exec_local: rm -f $${image_disk}.$${image_format} diff --git a/default/steps/bootstrap/prepare_ssh_to_out_context.yaml b/default/steps/bootstrap/prepare_ssh_to_out_context.yaml new file mode 100644 index 0000000..172f7a4 --- /dev/null +++ b/default/steps/bootstrap/prepare_ssh_to_out_context.yaml @@ -0,0 +1,23 @@ +- select_empty_port: + - on_checkpoint: redo + - exec_local: | + # Find empty SSH forwarding port + SSH_FWD_PORT=$(__find_free_port 50000 60000) + echo "SSH forwarding port: $SSH_FWD_PORT" +- prepare_ssh_config: + - on_checkpoint: redo + - write_local: + - $${ssh_config_file} + - | + Host $${kameleon_recipe_name} + HostName 127.0.0.1 + Port ${SSH_FWD_PORT} + User root + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + PasswordAuthentication no + IdentitiesOnly yes + LogLevel FATAL + ForwardAgent yes + Compression yes + Protocol 2 diff --git a/default/steps/bootstrap/start_http_server.yaml b/default/steps/bootstrap/start_http_server.yaml new file mode 100644 index 0000000..59184c3 --- /dev/null +++ b/default/steps/bootstrap/start_http_server.yaml @@ -0,0 +1,19 @@ +- http_script: $${kameleon_data_dir}/helpers/simple_http_server.py + +- run_http_server: + - exec_local: | + HTTP_PORT=$(__find_free_port 8000 8100) + echo "HTTP port: $HTTP_PORT" + export HTTP_PORT + - exec_local: python2 $${http_script} --root $${http_directory} --bind 0.0.0.0 --port $HTTP_PORT --daemon --pid $${http_pid} + - on_bootstrap_clean: + - exec_local: | + if [ -f $${http_pid} ]; then + HTTP_PID=$(cat $${http_pid}) + if ps -p $HTTP_PID > /dev/null; then + echo "Killing HTTP server (pid: $HTTP_PID)..." + kill -9 "$HTTP_PID" + rm -f $${http_pid} + fi + rm -f $${http_pid} + fi diff --git a/default/steps/bootstrap/start_qemu.yaml b/default/steps/bootstrap/start_qemu.yaml new file mode 100644 index 0000000..4d47953 --- /dev/null +++ b/default/steps/bootstrap/start_qemu.yaml @@ -0,0 +1,227 @@ +# Require SSH_FWD_PORT bash environment variable to be set + +# This must be set if you want to boot an ISO image: +- qemu_iso_path: "" +- qemu_iso_boot: true +# Else that can be set to boot from a kernel, initrd and cmdline: +- qemu_kernel_path: "" +- qemu_initrd_path: "" +- qemu_append_cmdline: "" +# Else boot from disk. + +- vm_expected_service: ssh +- boot_timeout: 100 +- shutdown_timeout: 100 +- debug: false +- telnet_port: "" +- no_reboot: true +- socat_monitor: socat - UNIX-CONNECT:$${qemu_monitor_socket} +- qemu_sendkeys_script: $${kameleon_data_dir}/qemu-sendkeys.rb +- qemu_sendkeys_commands: +- vm_expected_service: ssh +- vm_cleanup_section: setup +- shutdown_vm_immediately: false +- force_vm_shutdown: true +- qemu_enable_kvm: true +- qemu_cpus: 2 +- qemu_memory_size: 768 +- qemu_monitor_socket: $${kameleon_cwd}/qemu_monitor.socket +- qemu_arch: $${arch} +- qemu_image_size: 10G +- qemu_pidfile: $${kameleon_cwd}/qemu.pid +- qemu_uefi: false +- qemu_uefi_code_path: /usr/share/AAVMF/AAVMF_CODE.fd +- qemu_uefi_vars_path: /usr/share/AAVMF/AAVMF_VARS.fd +- qemu_netdev_user_options: +- disk_cache: unsafe + +- start_vm: + - on_checkpoint: redo + - check_cmd_local: qemu-system-$${qemu_arch} + - check_cmd_local: socat + - on_bootstrap_clean: + - test: + - exec_local: test "$${shutdown_vm_immediately}" == "false" -a "$${vm_cleanup_section}" == "bootstrap" + - group: + - exec_local: &1 | + if [ -f $${qemu_pidfile} ]; then + _QEMU_PID=$(< $${qemu_pidfile}) + if ps -p $_QEMU_PID > /dev/null; then + if [ "$${force_vm_shutdown}" == "true" ]; then + if [ -S $${qemu_monitor_socket} ]; then + echo "Executing a graceful shutdown of the qemu VM via the monitor socket..." + NEXT_WAIT_TIME=0 + echo system_powerdown | socat - UNIX-CONNECT:$${qemu_monitor_socket} || true + while ps -p $_QEMU_PID > /dev/null && [ $NEXT_WAIT_TIME -lt $${shutdown_timeout} ]; + do + sleep 1 + echo -en "\rWaiting for qemu virtual machine to shutdown...($(( $${shutdown_timeout} - 1 - NEXT_WAIT_TIME++ ))s)" + done + fi + else + echo "Waiting for the VM to shutdown" + echo "Run 'vncviewer :$VNC_PORT' to see what's happening in the VM" + while ps -p $_QEMU_PID > /dev/null; + do + sleep 2 + done + fi + fi + fi + - exec_local: &2 | + if [ -f $${qemu_pidfile} ]; then + _QEMU_PID=$(< $${qemu_pidfile}) + if ps -p $_QEMU_PID > /dev/null; then + if [ -S $${qemu_monitor_socket} ]; then + echo "The graceful shutdown of the qemu VM should have failed (monitor socket is there)..." + fi + echo "Killing qemu (pid: $_QEMU_PID)." + kill -9 "$_QEMU_PID" + fi + rm -f $${qemu_pidfile} + fi + rm -f $${qemu_monitor_socket} + - on_setup_clean: + - test: + - exec_local: test "$${shutdown_vm_immediately}" == "false" -a "$${vm_cleanup_section}" == "setup" + - group: + - exec_local: *1 + - exec_local: *2 + - on_export_clean: + - test: + - exec_local: test "$${shutdown_vm_immediately}" == "false" -a "$${vm_cleanup_section}" == "export" + - group: + - exec_local: *1 + - exec_local: *2 + - exec_local: | + if [ "$${shutdown_vm_immediately}" == "true" ]; then + echo "Qemu VM shutdown: immediately" + else + echo "Qemu VM shutdown: in $${vm_cleanup_section} section cleaning" + fi + - exec_local: | + if [ -r $${qemu_pidfile} ] && pgrep -F $${qemu_pidfile} > /dev/null; then + echo "Qemu pid file found, with process running: killing it !" 1>&2 + pkill -F $${qemu_pidfile} + sleep 0.5 + if pgrep -F $${qemu_pidfile} > /dev/null; then + echo "Failed to kill qemu process." 1>&2 + exit 1 + fi + fi + - exec_local: | + echo "Starting qemu..." + if [ "$${qemu_enable_kvm}" == "true" ] && (/usr/sbin/kvm-ok > /dev/null || egrep '(vmx|svm)' /proc/cpuinfo > /dev/null) ; then # print warning if /usr/sbin/kvm-ok is not installed + if [ "$${qemu_arch}" == "aarch64" ]; then + ENABLE_KVM="-enable-kvm -accel kvm -machine virt,gic-version=host,accel=kvm:tcg -cpu host" + #ENABLE_KVM="-global virtio-blk-pci.scsi=off -no-user-config -enable-fips -machine virt,gic-version=host,accel=kvm:tcg -cpu host -rtc driftfix=slew -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0" + elif [ "$${qemu_arch}" == "ppc64" ]; then + ENABLE_KVM="-enable-kvm -accel kvm -machine pseries,accel=kvm:tcg -cpu host" + else #X86_64 + ENABLE_KVM="-enable-kvm -cpu host" + fi + BOOT_TIMEOUT=$${boot_timeout} + else + echo "No KVM acceleration used" + BOOT_TIMEOUT=$(($${boot_timeout}*2)) + fi + if [ -f "vm_state_to_load.txt" ] + then + SAVED_STATE="$(< vm_state_to_load.txt)" + LOADVM="-loadvm $SAVED_STATE" + rm -f vm_state_to_load.txt + fi + if [ "$${debug}" == "true" ]; then + VNC_OPT="" + else + # Find empty VNC port + VNC_PORT=$(( $(__find_free_port 5900 5910) - 5900 )) + echo "VNC port: $VNC_PORT" + VNC_OPT="-vnc :$VNC_PORT" + fi + if [ -n "$${telnet_port}" ]; then + SERIAL_TELNET="telnet:localhost:$${telnet_port},server" + fi + # Select disk + QEMU_DRIVES="-drive file=$${image_disk}.$${image_format},cache=$${disk_cache},media=disk,if=virtio,id=drive0" + QEMU_BOOT= + QEMU_APPEND_CMDLINE= + if [ "$${qemu_uefi}" == "true" ]; then + if [ ! -f $${kameleon_cwd}/qemu_uefi_vars.fd ]; then + cp $${qemu_uefi_vars_path} $${kameleon_cwd}/qemu_uefi_vars.fd + fi + QEMU_BOOT="-drive if=pflash,format=raw,readonly,file=$${qemu_uefi_code_path} -drive if=pflash,format=raw,file=$${kameleon_cwd}/qemu_uefi_vars.fd" + fi + if [ -n "$${qemu_iso_path}" ]; then + QEMU_DRIVES="-drive file=$${qemu_iso_path},readonly,media=cdrom $QEMU_DRIVES" + if [ "$${qemu_iso_boot}" == "true" ]; then + QEMU_BOOT="$QEMU_BOOT -boot order=d" + fi + elif [ -n "$${qemu_kernel_path}" ]; then + QEMU_BOOT="$QEMU_BOOT -kernel $${qemu_kernel_path}" + if [ -n "$${qemu_initrd_path}" ]; then + QEMU_BOOT="$QEMU_BOOT -initrd $${qemu_initrd_path}" + fi + if [ -n "$${qemu_append_cmdline}" ]; then + QEMU_APPEND_CMDLINE="$${qemu_append_cmdline}" + QEMU_APPEND_CMDLINE=${QEMU_APPEND_CMDLINE//%LOCAL_IP%/$${local_ip}} + QEMU_APPEND_CMDLINE=${QEMU_APPEND_CMDLINE//%HTTP_PORT%/$HTTP_PORT} + fi + fi + if [ -n "$${qemu_netdev_user_options}" ]; then + QEMU_NETDEV_USER_OPTIONS=",$${qemu_netdev_user_options}" + fi + if [ "$${no_reboot}" == "true" ]; then + NO_REBOOT="-no-reboot" + fi + if [ -n "${SSH_FWD_PORT}" ]; then + HOSTFWD=",hostfwd=tcp::${SSH_FWD_PORT}-:22" + fi + qemu-system-$${qemu_arch} $ENABLE_KVM -smp $${qemu_cpus} -m $${qemu_memory_size} -rtc base=localtime \ + -net nic,model=virtio -net user${QEMU_NETDEV_USER_OPTIONS}${HOSTFWD} \ + $QEMU_DRIVES \ + -monitor unix:$${qemu_monitor_socket},server,nowait -pidfile $${qemu_pidfile} -daemonize \ + $QEMU_BOOT ${QEMU_APPEND_CMDLINE:+-append "$QEMU_APPEND_CMDLINE"} $NO_REBOOT \ + $VNC_OPT $SERIAL_TELNET\ + $LOADVM + - exec_local: | + VM_AVAILABLE=0 + if [ "$${vm_expected_service}" == "ssh" ]; then + TIMEOUT=$(( $(date +%s) + $BOOT_TIMEOUT )) + until timeout 5 ssh -q -F $${ssh_config_file} -o ConnectionAttempts=1 $${kameleon_recipe_name} -t true && VM_AVAILABLE=1 || [ $(date +%s) -gt $TIMEOUT ]; + do + echo -en "\rWaiting for SSH to become available in VM for out_context...($(( TIMEOUT - $(date +%s) ))s)" + sleep 1 + done + echo + else + TIMEOUT=$(( $(date +%s) + $BOOT_TIMEOUT )) + until timeout 1 [ $(date +%s) -gt $TIMEOUT ]; + do + echo -en "\rWaiting for VM to become available : ($(( TIMEOUT - $(date +%s) ))s)" + sleep 1 + done + echo + VM_AVAILABLE=1 + fi + - rescue: + - exec_local: test $VM_AVAILABLE -eq 1 + - breakpoint: | + Failed to get VM up and running (expected service: $${vm_expected_service}). Please verify the VM successfully booted with a vnc client. + - test: + - exec_local: test -e "$${qemu_sendkeys_commands}" -a -s "$${qemu_sendkeys_commands}" + - exec_local: | + echo "Sending keyboard commands to the VM: $${qemu_sendkeys_commands}" + echo "(Local httpd server url: http://$${local_ip}:$HTTP_PORT)" + ruby $${qemu_sendkeys_script} -d 0.05 "$(sed -e s/%LOCAL_IP%/$${local_ip}/g -e s/%HTTP_PORT%/$HTTP_PORT/g $${qemu_sendkeys_commands})" | $${socat_monitor} > /dev/null + - exec_local: echo "No keyboard commands to send" + +- shutdown_vm: + - on_checkpoint: redo + - on_clean: + - test: + - exec_local: test "$${shutdown_vm_immediately}" == "true" + - exec_local: *2 + - test: + - exec_local: test "$${shutdown_vm_immediately}" == "true" + - exec_local: *1 diff --git a/default/steps/checkpoints/simple.yaml b/default/steps/checkpoints/simple.yaml new file mode 100644 index 0000000..dbd60df --- /dev/null +++ b/default/steps/checkpoints/simple.yaml @@ -0,0 +1,21 @@ +enabled?: + - exec_local: test -f $${kameleon_cwd}/checkpoint_enabled + +create: + - exec_local: | + echo @microstep_id >> $${kameleon_cwd}/checkpoints.list + +apply: + - exec_local: | + touch $${kameleon_cwd}/checkpoints.list + grep -R @microstep_id $${kameleon_cwd}/checkpoints.list + + +clear: + - exec_local: | + echo > $${kameleon_cwd}/checkpoints.list + +list: + - exec_local: | + touch $${kameleon_cwd}/checkpoints.list + cat $${kameleon_cwd}/checkpoints.list | uniq diff --git a/default/steps/data/helpers/export_appliance.py b/default/steps/data/helpers/export_appliance.py new file mode 100644 index 0000000..634b240 --- /dev/null +++ b/default/steps/data/helpers/export_appliance.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +"""Convert a disk image to many others formats with guestfish.""" +from __future__ import division, unicode_literals + +import os +# import time +import os.path as op +import sys +import subprocess +import argparse +import logging + + +logger = logging.getLogger(__name__) + +tar_formats = ('tar', 'tar.gz', 'tgz', 'tar.bz2', 'tbz', 'tar.xz', 'txz', + 'tar.lzo', 'tzo') + +tar_options = ["--selinux", "--xattrs", "--xattrs-include=*", "--numeric-owner", "--one-file-system"] + +disk_formats = ('qcow', 'qcow2', 'qed', 'vdi', 'raw', 'vmdk') + + +def which(command): + """Locate a command. + Snippet from: http://stackoverflow.com/a/377028 + """ + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, fname = os.path.split(command) + if fpath: + if is_exe(command): + return command + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, command) + if is_exe(exe_file): + return exe_file + + raise ValueError("Command '%s' not found" % command) + + +def tar_convert(disk, output, excludes, compression_level): + """Convert image to a tar rootfs archive.""" + if compression_level in ("best", "fast"): + compression_level_opt = "--%s" % compression_level + else: + compression_level_opt = "-%s" % compression_level + + compr = "" + if output.endswith(('tar.gz', 'tgz')): + try: + compr = "| %s %s" % (which("pigz"), compression_level_opt) + except: + compr = "| %s %s" % (which("gzip"), compression_level_opt) + elif output.endswith(('tar.bz2', 'tbz')): + compr = "| %s %s" % (which("bzip2"), compression_level_opt) + elif output.endswith(('tar.xz', 'txz')): + compr = "| {} {} -c --threads=0 -".format( + which("xz"), compression_level_opt) + elif output.endswith(('tar.lzo', 'tzo')): + compr = "| %s %s -c -" % (which("lzop"), compression_level_opt) + + # NB: guestfish version >= 1.32 supports the special tar options, but not available in Debian stable (jessie): do not use for now + #tar_options_list = ["selinux:true", "acls:true", "xattrs:true", + # "numericowner:true", + # "excludes:\"%s\"" % ' '.join(excludes)] + #tar_options_str = ' '.join(tar_options_list) + #cmd = which("guestfish") + \ + # " --ro -i tar-out -a %s / - %s %s > %s" + #cmd = cmd % (disk, tar_options_str, compr, output) + #proc = subprocess.Popen(cmd_mount_tar, env=os.environ.copy(), shell=True) + #proc.communicate() + #if proc.returncode: + # raise subprocess.CalledProcessError(proc.returncode, cmd) + + tar_options_str = ' '.join(tar_options + ['--exclude="%s"' % s for s in excludes]) + # Necessary to have quick access to /etc (bug 12240) and also good for reproducibility + tar_options_str += ' --sort=name' + directory = dir_path = os.path.dirname(os.path.realpath(disk)) + cmds = [ + which("mkdir") + " %s/.mnt" % directory, + which("guestmount") + " --ro -i -a %s %s/.mnt" % (disk, directory), + which("tar") + " -c %s -C %s/.mnt . %s > %s" % (tar_options_str, directory, compr, output) + ] + cmd_mount_tar = " && ".join(cmds) + proc = subprocess.Popen(cmd_mount_tar, env=os.environ.copy(), shell=True) + proc.communicate() + returncode_mount_tar = proc.returncode + + # try to umount even if the previous command failed + cmds = [ + which("guestunmount") + " %s/.mnt" % directory, + which("rmdir") + " %s/.mnt" % directory + ] + cmd_umount = " && ".join(cmds) + proc = subprocess.Popen(cmd_umount, env=os.environ.copy(), shell=True) + proc.communicate() + returncode_umount = proc.returncode + + if returncode_mount_tar: + raise subprocess.CalledProcessError(returncode_mount_tar, cmd_mount_tar) + elif returncode_umount: + raise subprocess.CalledProcessError(returncode_umount, cmd_umount) + + +def qemu_convert(disk, output_fmt, output_filename): + """Convert the disk image filename to disk image output_filename.""" + binary = which("qemu-img") + cmd = [binary, "convert", "-O", output_fmt, disk, output_filename] + if output_fmt in ("qcow", "qcow2"): + cmd.insert(2, "-c") + proc = subprocess.Popen(cmd, env=os.environ.copy(), shell=False) + proc.communicate() + if proc.returncode: + raise subprocess.CalledProcessError(proc.returncode, ' '.join(cmd)) + + +def run_guestfish_script(disk, script, mount=""): + """ + Run guestfish script. + Mount should be in ("read_only", "read_write", "ro", "rw") + """ + args = [which("guestfish"), '-a', disk] + if mount in ("read_only", "read_write", "ro", "rw"): + args.append('-i') + if mount in mount in ("read_only", "ro"): + args.append('--ro') + else: + args.append('--rw') + else: + script = "run\n%s" % script + proc = subprocess.Popen(args, + stdin=subprocess.PIPE, + env=os.environ.copy()) + proc.communicate(input=script.encode('utf-8')) + if proc.returncode: + raise subprocess.CalledProcessError(proc.returncode, ' '.join(args)) + + +def guestfish_zerofree(filename): + """Fill free space with zero""" + logger.info(guestfish_zerofree.__doc__) + cmd = "virt-filesystems -a %s" % filename + fs = subprocess.check_output(cmd.encode('utf-8'), + stderr=subprocess.STDOUT, + shell=True, + env=os.environ.copy()) + list_fs = fs.decode('utf-8').split() + logger.info('\n'.join((' `--> %s' % i for i in list_fs))) + script = '\n'.join(('zerofree %s' % i for i in list_fs)) + run_guestfish_script(filename, script, mount="read_only") + + +def convert_disk_image(args): + """Convert disk to another format.""" + filename = op.abspath(args.file.name) + output = op.abspath(args.output) + + os.environ['LIBGUESTFS_CACHEDIR'] = os.getcwd() + if args.verbose: + os.environ['LIBGUESTFS_DEBUG'] = '1' + + # sometimes guestfish fails because of other virtualization tools are + # still running use a test and retry to wait for availability + # attempts = 0 + # while attempts < 3: + # try: + # logger.info("Waiting for virtualisation to be available...") + # run_guestfish_script(filename, "cat /etc/hostname", mount='ro') + # break + # except: + # attempts += 1 + # time.sleep(1) + + if args.zerofree and (set(args.formats) & set(disk_formats)): + guestfish_zerofree(filename) + + for fmt in args.formats: + if fmt in (tar_formats + disk_formats): + output_filename = "%s.%s" % (output, fmt) + if output_filename == filename: + continue + logger.info("Creating %s" % output_filename) + try: + if fmt in tar_formats: + tar_convert(filename, output_filename, + args.tar_excludes, + args.tar_compression_level) + else: + qemu_convert(filename, fmt, output_filename) + except ValueError as exp: + logger.error("Error: %s" % exp) + + +if __name__ == '__main__': + allowed_formats = tar_formats + disk_formats + allowed_formats_help = 'Allowed values are ' + ', '.join(allowed_formats) + + allowed_levels = ["%d" % i for i in range(1, 10)] + ["best", "fast"] + allowed_levels_helps = 'Allowed values are ' + ', '.join(allowed_levels) + + parser = argparse.ArgumentParser( + description=sys.modules[__name__].__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument('file', action="store", type=argparse.FileType('r'), + help='Disk image filename') + parser.add_argument('-F', '--formats', action="store", type=str, nargs='+', + help='Output format. ' + allowed_formats_help, + choices=allowed_formats, metavar='fmt', required=True) + parser.add_argument('-o', '--output', action="store", type=str, + help='Output filename (without file extension)', + required=True, metavar='filename') + parser.add_argument('--tar-compression-level', action="store", type=str, + default="9", choices=allowed_levels, metavar='lvl', + help="Compression level. " + allowed_levels_helps) + parser.add_argument('--tar-excludes', action="store", type=str, nargs='+', + help="Files to excluded from archive", + metavar='pattern', default=[]) + parser.add_argument('--zerofree', action="store_true", default=False, + help='Zero free unallocated blocks from ext2/3 ' + 'file-systems before export to reduce image size') + parser.add_argument('--verbose', action="store_true", default=False, + help='Enable very verbose messages') + log_format = '%(levelname)s: %(message)s' + level = logging.INFO + args = parser.parse_args() + if args.verbose: + level = logging.DEBUG + + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(level) + handler.setFormatter(logging.Formatter(log_format)) + + logger.setLevel(level) + logger.addHandler(handler) + + convert_disk_image(args) diff --git a/default/steps/data/helpers/netinstall_iso_finder.py b/default/steps/data/helpers/netinstall_iso_finder.py new file mode 100644 index 0000000..b4a135b --- /dev/null +++ b/default/steps/data/helpers/netinstall_iso_finder.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +"""Find the latest netinstall iso for a Debian version and system architecture.""" + +from html.parser import HTMLParser +from urllib2 import urlopen +from urlparse import urljoin +import re +import sys +import argparse +import logging + +logger = logging.getLogger(__name__) + +class LinkParser(HTMLParser): + """Retrieve links (a hrefs) from a text/html document""" + def __init__(self, url): + HTMLParser.__init__(self) + self.url = url + self.links = set() + response = urlopen(url) + contentType = response.info().get('Content-Type') + if not contentType: + return + logger.debug("url = " + url ); + logger.debug("contentType = " + contentType ); + if ';' in contentType: + (mediaType,charset) = contentType.split(";") + charset = charset.split("=")[1] + else: + mediaType = contentType + # ISO-8859-1 is no longer the default charset, see https://tools.ietf.org/html/rfc7231#appendix-B + # Let's use UTF-8. + charset = "utf-8" + if mediaType =='text/html': + htmlBytes = response.read() + htmlString = htmlBytes.decode(charset) + self.feed(htmlString) + + def handle_starttag(self, tag, attrs): + if tag == 'a': + for (key, value) in attrs: + if key == 'href': + new_url = urljoin(self.url,value) + if re.match("^"+self.url, new_url): + self.links.add(new_url) + + def get_links(self): + """Returns all the collected links""" + return self.links + + +def url_find(to_visit_url_set,visited_url_set,found_url_set): + """Recursively look for urls given a regex, a set of urls to visit, a set of already visited urls, a set of already found urls. Returns the set of found urls""" + logger.debug("Progress: to_visit:{} visited:{} found:{}".format(len(to_visit_url_set),len(visited_url_set),len(found_url_set))) + assert(len(to_visit_url_set.intersection(visited_url_set)) == 0) + assert(len(to_visit_url_set.intersection(found_url_set)) == 0) + if (len(to_visit_url_set) == 0): + return [visited_url_set,found_url_set] + else: + url = to_visit_url_set.pop() + visited_url_set.add(url) + if target_regex.match(url): + found_url_set.add(url) + return url_find(to_visit_url_set, visited_url_set, found_url_set) + else: + new_url_set = set([url for url in LinkParser(url).get_links() if (logger.debug(url) or True) and url_regex.match(url)]) + new_url_set.difference_update(visited_url_set) + to_visit_url_set.update(new_url_set) + return url_find(to_visit_url_set, visited_url_set, found_url_set) + +def key_normalize(version_string): + """" + In order to perform a natural sorting, we normalize the version (X.Y.Z) as a unique integer with the following formula: X*100 + Y*10 + Z + For instance, it solves situations where "9.9.0" is greater than "9.9.11" + """ + splitted_string = version_string.split('.') + assert(len(splitted_string) == 3) + return int(splitted_string[0])*100+int(splitted_string[1])*10+int(splitted_string[2]) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("distrib", metavar="DISTRIB", help="distribution") + parser.add_argument("version", metavar="VERSION", help="version") + parser.add_argument("arch", metavar="ARCH", help="architecture") + parser.add_argument("mirror", metavar="MIRROR", help="mirror", nargs="?") + parser.add_argument('--info', action="store_true", default=False, help='print info messages') + parser.add_argument('--debug', action="store_true", default=False, help='print debug messages') + args = parser.parse_args() + + handler = logging.StreamHandler() + if args.debug: + logger.setLevel(logging.DEBUG) + handler.setLevel(logging.DEBUG) + elif args.info: + logger.setLevel(logging.INFO) + handler.setLevel(logging.INFO) + else: + logger.setLevel(logging.WARNING) + handler.setLevel(logging.WARNING) + handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) + logger.addHandler(handler) + + try: + visited = set([]) + found = set([]) + if (args.distrib.lower() == "debian"): + if args.mirror == None: + args.mirror = "http://cdimage.debian.org/" + if not re.match("^\d+$",args.version): + raise Exception("please give the Debian release number (e.g. 8 for Jessie)") + if args.version == '10': + url_regex = re.compile("^"+args.mirror+"cdimage/release/(?:"+args.version+"\.\d+\.\d+/(?:"+args.arch+"/(?:iso-cd/(?:debian-"+args.version+"\.\d+\.\d+-"+args.arch+"-netinst\.iso)?)?)?)?$") + else: + url_regex = re.compile("^"+args.mirror+"cdimage/archive/(?:"+args.version+"\.\d+\.\d+/(?:"+args.arch+"/(?:iso-cd/(?:debian-"+args.version+"\.\d+\.\d+-"+args.arch+"-netinst\.iso)?)?)?)?$") + target_regex = re.compile("^.*-netinst\.iso$") + [visited,found] = url_find(set([args.mirror+"cdimage/"+v+"/" for v in ["release","archive"]]), set(), set()) + elif (args.distrib.lower() == "ubuntu"): + if args.mirror == None: + args.mirror = "http://(?:archive|old-releases).ubuntu.com/" + servers = set(["http://"+s+".ubuntu.com/ubuntu/" for s in ["old-releases","archive"]]) + else: + servers = set([args.mirror]) + if not re.match("^\w+$",args.version): + raise Exception("please give the Ubuntu release name") + url_regex = re.compile("^"+args.mirror+"ubuntu/dists/(?:"+args.version+"(?:-updates)?/(?:main/(?:installer-"+args.arch+"/(?:current/(?:(?:legacy-)?images/(?:netboot/(?:mini\.iso)?)?)?)?)?)?)?$") + target_regex = re.compile("^.*/mini\.iso$") + [visited,found] = url_find(servers, set(), set()) + elif (args.distrib.lower() == "centos"): + if args.mirror == None: + args.mirror = "http://mirror.in2p3.fr/linux/CentOS/" + if not re.match("^\d+$",args.version): + raise Exception("please give the CentOS release number (e.g. 7 for CentOS-7)") + if args.version == '6': + url_regex = re.compile("^"+args.mirror+"(?:"+args.version+"/(?:isos/(?:"+args.arch+"/(?:CentOS-"+args.version+"(?:\.\d+)?-"+args.arch+"-netinstall\.iso)?)?)?)?$") + target_regex = re.compile("^.*CentOS-\d+(?:\.\d+)?-\w+-netinstall\.iso$") + elif args.version == '7': + url_regex = re.compile("^"+args.mirror+"(?:"+args.version+"/(?:isos/(?:"+args.arch+"/(?:CentOS-"+args.version+"-"+args.arch+"-NetInstall-\d+\.iso)?)?)?)?$") + target_regex = re.compile("^.*CentOS-\d+-\w+-NetInstall-\d+\.iso$") + else: + url_regex = re.compile("^"+args.mirror+"(?:"+args.version+"/(?:isos/(?:"+args.arch+"/(?:CentOS-"+args.version+"\.\d+\.\d+-"+args.arch+"-boot\.iso)?)?)?)?$") + target_regex = re.compile("^.*CentOS-\d+\.\d+\.\d+-\w+-boot\.iso$") + [visited,found] = url_find(set([args.mirror]), set(), set()) + else: + raise Exception("this distribution is not supported") + logger.info("URL regex: "+url_regex.pattern) + logger.info("Target regex: "+target_regex.pattern) + logger.debug("Visited URLs:") + for url in visited: + logger.debug(url) + logger.info("Found URLs:") + for url in found: + logger.info(url) + if len(found) > 0: + if (args.distrib.lower() == "debian"): + print(sorted(found,key=lambda x:key_normalize(re.sub(r".*/debian-(\d+).(\d+).(\d+)-"+args.arch+"-netinst\.iso$",r"\1.\2.\3",x)),reverse=True)[0]) + else: + print(sorted(found, reverse=False)[0]) + else: + raise Exception("no url found") + except Exception as exc: + sys.stderr.write(u"Error: %s\n" % exc) + sys.exit(1) diff --git a/default/steps/data/helpers/simple_http_server.py b/default/steps/data/helpers/simple_http_server.py new file mode 100644 index 0000000..881343a --- /dev/null +++ b/default/steps/data/helpers/simple_http_server.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python2 +"""Simple HTTP server""" +from __future__ import unicode_literals +import atexit +import os +import sys +import argparse + + +class HTTPServerDaemon(object): + + """A HTTP server daemon class.""" + + def __init__(self, root=os.getcwd()): + """ Initialize the object.""" + self.root = root + + def daemonize(self, pidfile): + """Deamonize class. UNIX double fork mechanism.""" + try: + pid = os.fork() + if pid > 0: + # exit first parent + sys.exit(0) + except OSError as err: + sys.stderr.write('fork #1 failed: {0}\n'.format(err)) + sys.exit(1) + + # decouple from parent environment + os.chdir(self.root) + os.setsid() + os.umask(0) + + # do second fork + try: + pid = os.fork() + if pid > 0: + + # exit from second parent + sys.exit(0) + except OSError as err: + sys.stderr.write('fork #2 failed: {0}\n'.format(err)) + sys.exit(1) + + # redirect standard file descriptors + sys.stdout.flush() + sys.stderr.flush() + si = open(os.devnull, 'r') + so = open(os.devnull, 'a+') + se = open(os.devnull, 'a+') + + os.dup2(si.fileno(), sys.stdin.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) + os.dup2(se.fileno(), sys.stderr.fileno()) + + # Make sure pid file is removed if we quit + @atexit.register + def delpid(self): + os.remove(pidfile) + + # write pidfile + pid = str(os.getpid()) + with open(pidfile, 'w+') as f: + f.write(pid + '\n') + + def start(self, pidfile, *args, **kwargs): + """Start the daemon.""" + # Check for a pidfile to see if the daemon already runs + try: + with open(pidfile, 'r') as pf: + + pid = int(pf.read().strip()) + except IOError: + pid = None + + if pid: + message = "pidfile {0} already exist. " + \ + "Daemon already running?\n" + sys.stderr.write(message.format(pidfile)) + sys.exit(1) + + # Start the daemon + self.daemonize(pidfile) + self.run(*args, **kwargs) + + def run(self, host, port): + """ Run an HTTP server.""" + if sys.version_info[0] == 3: + from http.server import HTTPServer, SimpleHTTPRequestHandler + httpd = HTTPServer((host, port), SimpleHTTPRequestHandler) + else: + import SimpleHTTPServer + import SocketServer + handler = SimpleHTTPServer.SimpleHTTPRequestHandler + httpd = SocketServer.TCPServer((host, port), handler) + + print("Running on http://%s:%s/" % (host, port)) + os.chdir(self.root) + try: + httpd.serve_forever() + except KeyboardInterrupt: + sys.stderr.write(u"\nBye\n") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=sys.modules[__name__].__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument('--port', action="store", default=9090, type=int, + help='Set the listening port') + parser.add_argument('--root', action="store", default=os.getcwd()) + parser.add_argument('--bind', action="store", default="0.0.0.0", + help='Set the binding address') + parser.add_argument('--daemon', action="store_true", default=False) + parser.add_argument('--pid', action="store") + + try: + args = parser.parse_args() + http_server = HTTPServerDaemon(root=args.root) + if args.daemon: + if args.pid is None: + parser.error("Need to set a pid file") + http_server.start(args.pid, args.bind, args.port) + else: + http_server.run(args.bind, args.port) + except Exception as exc: + sys.stderr.write(u"\nError: %s\n" % exc) + sys.exit(1) diff --git a/default/steps/data/preseed/debian-testing-preseed.cfg b/default/steps/data/preseed/debian-testing-preseed.cfg new file mode 100644 index 0000000..5af0d99 --- /dev/null +++ b/default/steps/data/preseed/debian-testing-preseed.cfg @@ -0,0 +1,322 @@ +#### Contents of the preconfiguration file (for wheezy) +### Localization +# Locale sets language and country. +d-i debian-installer/locale string en_US.UTF-8 + +# Keyboard selection. +#d-i keymap select us +d-i keyboard-configuration/xkb-keymap select us + +### Network configuration +# netcfg will choose an interface that has link if possible. This makes it +# skip displaying a list if there is more than one interface. +d-i netcfg/choose_interface select auto + +# To pick a particular interface instead: +#d-i netcfg/choose_interface select eth1 + +# If you have a slow dhcp server and the installer times out waiting for +# it, this might be useful. +#d-i netcfg/dhcp_timeout string 60 + +# If you prefer to configure the network manually, uncomment this line and +# the static network configuration below. +#d-i netcfg/disable_dhcp boolean true + +# If you want the preconfiguration file to work on systems both with and +# without a dhcp server, uncomment these lines and the static network +# configuration below. +#d-i netcfg/dhcp_failed note +#d-i netcfg/dhcp_options select Configure network manually + +# Static network configuration. +#d-i netcfg/get_nameservers string 192.168.1.1 +#d-i netcfg/get_ipaddress string 192.168.1.42 +#d-i netcfg/get_netmask string 255.255.255.0 +#d-i netcfg/get_gateway string 192.168.1.1 +#d-i netcfg/confirm_static boolean true + +# Any hostname and domain names assigned from dhcp take precedence over +# values set here. However, setting the values still prevents the questions +# from being shown, even if values come from dhcp. +d-i netcfg/get_hostname string kameleon +d-i netcfg/get_domain string kameleon + +# Disable that annoying WEP key dialog. +d-i netcfg/wireless_wep string +# The wacky dhcp hostname that some ISPs use as a password of sorts. +#d-i netcfg/dhcp_hostname string radish + +# If non-free firmware is needed for the network or other hardware, you can +# configure the installer to always try to load it, without prompting. Or +# change to false to disable asking. +#d-i hw-detect/load_firmware boolean true + +### Network console +# Use the following settings if you wish to make use of the network-console +# component for remote installation over SSH. This only makes sense if you +# intend to perform the remainder of the installation manually. +#d-i anna/choose_modules string network-console +#d-i network-console/password password r00tme +#d-i network-console/password-again password r00tme + +### Mirror settings +# If you select ftp, the mirror/country string does not need to be set. +#d-i mirror/protocol string ftp +d-i mirror/country string manual +d-i mirror/http/hostname string http.debian.net +d-i mirror/http/directory string /debian +d-i mirror/http/proxy string + +# Suite to install. +d-i mirror/suite string testing +# Suite to use for loading installer components (optional). +d-i mirror/udeb/suite string unstable + +### Clock and time zone setup +# Controls whether or not the hardware clock is set to UTC. +d-i clock-setup/utc boolean true + +# You may set this to any valid setting for $TZ; see the contents of +# /usr/share/zoneinfo/ for valid values. +d-i time/zone string UTC + +# Controls whether to use NTP to set the clock during the install +d-i clock-setup/ntp boolean true +# NTP server to use. The default is almost always fine here. +#d-i clock-setup/ntp-server string ntp.example.com + +### Partitioning +# If the system has free space you can choose to only partition that space. +#d-i partman-auto/init_automatically_partition select biggest_free + +# Alternatively, you can specify a disk to partition. The device name must +# be given in traditional non-devfs format. +# Note: A disk must be specified, unless the system has only one disk. +# For example, to use the first SCSI/SATA hard disk: +#d-i partman-auto/disk string /dev/sda +# In addition, you'll need to specify the method to use. +# The presently available methods are: "regular", "lvm" and "crypto" +d-i partman-auto/method string regular + +# If one of the disks that are going to be automatically partitioned +# contains an old LVM configuration, the user will normally receive a +# warning. This can be preseeded away... +d-i partman-lvm/device_remove_lvm boolean true +# The same applies to pre-existing software RAID array: +d-i partman-md/device_remove_md boolean true + +# And the same goes for the confirmation to write the lvm partitions. +d-i partman-lvm/confirm boolean true +d-i partman-lvm/confirm_nooverwrite boolean true + + +d-i partman/choose_partition select finish +d-i partman-auto-lvm/guided_size string max + +# You can choose one of the three predefined partitioning recipes: +# - atomic: all files in one partition +# - home: separate /home partition +# - multi: separate /home, /usr, /var, and /tmp partitions +d-i partman-auto/choose_recipe select atomic +d-i partman/default_filesystem string ext4 + +# Or provide a recipe of your own... +# The recipe format is documented in the file devel/partman-auto-recipe.txt. +# If you have a way to get a recipe file into the d-i environment, you can +# just point at it. +#d-i partman-auto/expert_recipe_file string /hd-media/recipe + +# If not, you can put an entire recipe into the preconfiguration file in one +# (logical) line. This example creates a small /boot partition, suitable +# swap, and uses the rest of the space for the root partition: +#d-i partman-auto/expert_recipe string \ +# boot-root :: \ +# 40 50 100 ext3 \ +# $primary{ } $bootable{ } \ +# method{ format } format{ } \ +# use_filesystem{ } filesystem{ ext3 } \ +# mountpoint{ /boot } \ +# . \ +# 500 10000 1000000000 ext3 \ +# method{ format } format{ } \ +# use_filesystem{ } filesystem{ ext3 } \ +# mountpoint{ / } \ +# . \ +# 64 512 300% linux-swap \ +# method{ swap } format{ } \ +# . + +#The preseed line that "selects finish" needs to be in a certain order in your preseed, the example-preseed does not follow this. +#http://ubuntuforums.org/archive/index.php/t-1504045.html + +# This makes partman automatically partition without confirmation, provided +# that you told it what to do using one of the methods above. +d-i partman/confirm_write_new_label boolean true +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true + + +### Base system installation +# Select the initramfs generator used to generate the initrd for 2.6 kernels. +#d-i base-installer/kernel/linux/initramfs-generators string yaird + +# The kernel image (meta) package to be installed; "none" can be used if no +# kernel is to be installed. +#d-i base-installer/kernel/image string linux-image-2.6-486 + +### Account setup +# Enable login to root account +d-i passwd/root-login boolean true +# Root password, either in clear text +d-i passwd/root-password password kameleon +d-i passwd/root-password-again password kameleon +# or encrypted using an MD5 hash. +#d-i passwd/root-password-crypted password [MD5 hash] + +# Skip creation of a normal user account. +# d-i passwd/make-user boolean false + +# To create a normal user account. +d-i passwd/user-fullname string Kameleon User +d-i passwd/username string kameleon +# Normal user's password, either in clear text +d-i passwd/user-password password kameleon +d-i passwd/user-password-again password kameleon +# or encrypted using an MD5 hash. +#d-i passwd/user-password-crypted password [MD5 hash] +# Create the first user with the specified UID instead of the default. +#d-i passwd/user-uid string 1010 +# d-i user-setup/encrypt-home boolean false +# d-i user-setup/allow-password-weak boolean true + +# The user account will be added to some standard initial groups. To +# override that, use this. +d-i passwd/user-default-groups string audio cdrom video admin + +### Apt setup +# You can choose to install non-free and contrib software. +#d-i apt-setup/non-free boolean true +#d-i apt-setup/contrib boolean true +# Uncomment this if you don't want to use a network mirror. +#d-i apt-setup/use_mirror boolean false +# Select which update services to use; define the mirrors to be used. +# Values shown below are the normal defaults. +# FIXME : temporarily remove security repo while debian fixes the installer (default value : d-i apt-setup/services-select multiselect security, volatile) +d-i apt-setup/services-select multiselect +#d-i apt-setup/security_host string security.debian.org +#d-i apt-setup/volatile_host string volatile.debian.org + +# Scan another CD or DVD? +d-i apt-setup/cdrom/set-first boolean false + +# By default the installer requires that repositories be authenticated +# using a known gpg key. This setting can be used to disable that +# authentication. Warning: Insecure, not recommended. +#d-i debian-installer/allow_unauthenticated string true + +### Package selection +tasksel tasksel/first multiselect none +# If the desktop task is selected, install the kde and xfce desktops +# instead of the default gnome desktop. +#tasksel tasksel/desktop multiselect kde, xfce + +# Individual additional packages to install +d-i pkgsel/include string openssh-server sudo rsync haveged + +# Whether to upgrade packages after debootstrap. +# Allowed values: none, safe-upgrade, full-upgrade +d-i pkgsel/upgrade select none + +# Some versions of the installer can report back on what software you have +# installed, and what software you use. The default is not to report back, +# but sending reports helps the project determine what software is most +# popular and include it on CDs. +popularity-contest popularity-contest/participate boolean false + +### Boot loader installation +# Grub is the default boot loader (for x86). If you want lilo installed +# instead, uncomment this: +#d-i grub-installer/skip boolean true +# To also skip installing lilo, and install no bootloader, uncomment this +# too: +#d-i lilo-installer/skip boolean true + +# This is fairly safe to set, it makes grub install automatically to the MBR +# if no other operating system is detected on the machine. +d-i grub-installer/only_debian boolean true + +# This one makes grub-installer install to the MBR if it also finds some other +# OS, which is less safe as it might not be able to boot that other OS. +d-i grub-installer/with_other_os boolean true + +# Alternatively, if you want to install to a location other than the mbr, +# uncomment and edit these lines: +#d-i grub-installer/only_debian boolean false +#d-i grub-installer/with_other_os boolean false +#d-i grub-installer/bootdev string (hd0,0) +# To install grub to multiple disks: +#d-i grub-installer/bootdev string (hd0,0) (hd1,0) (hd2,0) + +# Optional password for grub, either in clear text +#d-i grub-installer/password password r00tme +#d-i grub-installer/password-again password r00tme +# or encrypted using an MD5 hash, see grub-md5-crypt(8). +#d-i grub-installer/password-crypted password [MD5 hash] + +# GRUB install devices: +# Choices: /dev/sda (21474 MB; VMware_Virtual_S), /dev/sda1 (21472 MB; VMware_Virtual_S) +grub-pc grub-pc/install_devices multiselect /dev/vda +# Choices: Enter device manually, /dev/sda +grub-installer grub-installer/choose_bootdev select /dev/vda + +### Finishing up the installation +# During installations from serial console, the regular virtual consoles +# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next +# line to prevent this. +#d-i finish-install/keep-consoles boolean true + +# Avoid that last message about the install being complete. +d-i finish-install/reboot_in_progress note + +# This will prevent the installer from ejecting the CD during the reboot, +# which is useful in some situations. +d-i cdrom-detect/eject boolean false + +# This is how to make the installer shutdown when finished, but not +# reboot into the installed system. +#d-i debian-installer/exit/halt boolean true +# This will power off the machine instead of just halting it. +d-i debian-installer/exit/poweroff boolean true + +### Preseeding other packages +# Depending on what software you choose to install, or if things go wrong +# during the installation process, it's possible that other questions may +# be asked. You can preseed those too, of course. To get a list of every +# possible question that could be asked during an install, do an +# installation, and then run these commands: +# debconf-get-selections --installer > file +# debconf-get-selections >> file + + +#### Advanced options +### Running custom commands during the installation +# d-i preseeding is inherently not secure. Nothing in the installer checks +# for attempts at buffer overflows or other exploits of the values of a +# preconfiguration file like this one. Only use preconfiguration files from +# trusted locations! To drive that home, and because it's generally useful, +# here's a way to run any shell command you'd like inside the installer, +# automatically. + +# This first command is run as early as possible, just after +# preseeding is read. +# Prevent packaged version of VirtualBox Guest Additions being installed: +#d-i preseed/early_command string sed -i \ +# '/in-target/idiscover(){/sbin/discover|grep -v VirtualBox;}' \ +# /usr/lib/pre-pkgsel.d/20install-hwpackages + +# This command is run just before the install finishes, but when there is +# still a usable /target directory. You can chroot to /target and use it +# directly, or use the apt-install and in-target commands to easily install +# packages and run commands in the target system. + diff --git a/default/steps/data/qemu-sendkeys.rb b/default/steps/data/qemu-sendkeys.rb new file mode 100644 index 0000000..d1bcb0f --- /dev/null +++ b/default/steps/data/qemu-sendkeys.rb @@ -0,0 +1,121 @@ +#!/usr/bin/env ruby +# Translate a string to "sendkey" commands for QEMU. +# Martin Vidner, MIT License + +# https://en.wikibooks.org/wiki/QEMU/Monitor#sendkey_keys +# sendkey keys +# +# You can emulate keyboard events through sendkey command. The syntax is: sendkey keys. To get a list of keys, type sendkey [tab]. Examples: +# +# sendkey a +# sendkey shift-a +# sendkey ctrl-u +# sendkey ctrl-alt-f1 +# +# As of QEMU 0.12.5 there are: +# shift shift_r alt alt_r altgr altgr_r +# ctrl ctrl_r menu esc 1 2 +# 3 4 5 6 7 8 +# 9 0 minus equal backspace tab +# q w e r t y +# u i o p ret a +# s d f g h j +# k l z x c v +# b n m comma dot slash +# asterisk spc caps_lock f1 f2 f3 +# f4 f5 f6 f7 f8 f9 +# f10 num_lock scroll_lock kp_divide kp_multiply kp_subtract +# kp_add kp_enter kp_decimal sysrq kp_0 kp_1 +# kp_2 kp_3 kp_4 kp_5 kp_6 kp_7 +# kp_8 kp_9 < f11 f12 print +# home pgup pgdn end left up +# down right insert delete + +require "optparse" + +# incomplete! only what I need now. +KEYS = { + "%" => "shift-5", + "/" => "slash", + ":" => "shift-semicolon", + "=" => "equal", + "." => "dot", + " " => "spc", + "-" => "minus", + "_" => "shift-minus", + "*" => "asterisk", + "," => "comma", + "+" => "shift-equal", + "|" => "shift-backslash", + "\\" => "backslash", +} + +class Main + attr_accessor :command + attr_accessor :delay_s + attr_accessor :keystring + + def initialize + self.command = nil + self.delay_s = 0.1 + + OptionParser.new do |opts| + opts.banner = "Usage: sendkeys [-c command_to_pipe_to] STRING\n" + + "Where STRING can be 'ls<enter>ls<gt>/dev/null<enter>'" + + opts.on("-c", "--command COMMAND", + "Pipe sendkeys to this commands, individually") do |v| + self.command = v + end + opts.on("-d", "--delay SECONDS", Float, + "Delay SECONDS after each key (default: 0.1)") do |v| + self.delay_s = v + end + end.parse! + self.keystring = ARGV[0] + end + + def sendkey(qemu_key_name) + if qemu_key_name == "wait" + sleep 1 + else + if qemu_key_name =~ /[A-Za-z]/ && qemu_key_name == qemu_key_name.upcase + key = "shift-#{qemu_key_name.downcase}" + else + key = qemu_key_name + end + qemu_cmd = "sendkey #{key}" + if command + system "echo '#{qemu_cmd}' | #{command}" + else + puts qemu_cmd + $stdout.flush # important when we are piped + end + sleep delay_s + end + end + + PATTERN = / + \G # where last match ended + < [^>]+ > + | + \G + . + /x + def run + keystring.scan(PATTERN) do |match| + if match[0] == "<" + key_name = match.slice(1..-2) + sendkey case key_name + when "lt" then "shift-comma" + when "gt" then "shift-dot" + else key_name + end + else + sendkey KEYS.fetch(match, match) + end + end + end +end + +Main.new.run diff --git a/default/steps/data/qemu-sendkeys/netinst-iso-debian b/default/steps/data/qemu-sendkeys/netinst-iso-debian new file mode 100644 index 0000000..7705a44 --- /dev/null +++ b/default/steps/data/qemu-sendkeys/netinst-iso-debian @@ -0,0 +1 @@ +<esc><wait>auto preseed/url=http://%LOCAL_IP%:%HTTP_PORT%/preseed.cfg<kp_enter> diff --git a/default/steps/disable_checkpoint.yaml b/default/steps/disable_checkpoint.yaml new file mode 100644 index 0000000..cb571da --- /dev/null +++ b/default/steps/disable_checkpoint.yaml @@ -0,0 +1,3 @@ +- disable_checkpoint: + - on_checkpoint: redo + - exec_local: rm -f $${kameleon_cwd}/checkpoint_enabled diff --git a/default/steps/enable_checkpoint.yaml b/default/steps/enable_checkpoint.yaml new file mode 100644 index 0000000..8ac4751 --- /dev/null +++ b/default/steps/enable_checkpoint.yaml @@ -0,0 +1,5 @@ +- enable_checkpoint: + - on_checkpoint: redo + - on_bootstrap_init: + - exec_local: rm -f $${kameleon_cwd}/checkpoint_enabled + - exec_local: touch $${kameleon_cwd}/checkpoint_enabled diff --git a/default/steps/env/bashrc b/default/steps/env/bashrc new file mode 100644 index 0000000..6306e37 --- /dev/null +++ b/default/steps/env/bashrc @@ -0,0 +1,23 @@ +## aliases +# If not running interactively, don't do anything +export USER=${USER:-"root"} +export HOME=${HOME:-"/root"} +export PATH=/usr/bin:/usr/sbin:/bin:/sbin:$PATH +export LC_ALL=${LC_ALL:-"POSIX"} + +export DEBIAN_FRONTEND=noninteractive + +if [ -t 1 ] ; then +export TERM=xterm +# for fast typing +alias h='history' +alias g='git status' +alias l='ls -lah' +alias ll='ls -lh' +alias la='ls -Ah' + +# for human readable output +alias ls='ls -h' +alias df='df -h' +alias du='du -h' +fi diff --git a/default/steps/env/functions.sh b/default/steps/env/functions.sh new file mode 100644 index 0000000..1abcc38 --- /dev/null +++ b/default/steps/env/functions.sh @@ -0,0 +1,201 @@ +## functions + +function fail { + echo $@ 1>&2 + false +} + +export -f fail + +function __download { + local src=$1 + local dst=$2 + if [ -n "$DOWNLOAD_SRC_URL" ]; then + src="$DOWNLOAD_SRC_URL" + fi + if [ -z "$src" ]; then + fail "No URL to download from" + fi + # If dst is unset or a directory, infers dst pathname from src + if [ -z "$dst" -o "${dst: -1}" == "/" ]; then + dst="$dst${src##*/}" + dst="${dst%%\?*}" + fi + local dstdir=${dst%/*} + if [ -n "$dstdir" -a "$dstdir" != "$dst" ]; then + mkdir -p $dstdir + fi + echo -n "Downloading: $src..." + # Put cURL first because it accept URIs (like file://...) + if which curl >/dev/null; then + echo " (cURL)" + curl -S --fail -# -L --retry 999 --retry-max-time 0 "$src" -o "$dst" 2>&1 + elif which wget >/dev/null; then + echo " (wget)" + wget --retry-connrefused --progress=bar:force "$src" -O "$dst" 2>&1 + elif which python >/dev/null; then + echo " (python)" + python -c <<EOF +import sys +import time +if sys.version_info >= (3,): + import urllib.request as urllib +else: + import urllib + + +def reporthook(count, block_size, total_size): + global start_time + if count == 0: + start_time = time.time() + return + duration = time.time() - start_time + progress_size = float(count * block_size) + if duration != 0: + if total_size == -1: + total_size = block_size + percent = 'Unknown size, ' + else: + percent = '%.0f%%, ' % float(count * block_size * 100 / total_size) + speed = int(progress_size / (1024 * duration)) + sys.stdout.write('\r%s%.2f MB, %d KB/s, %d seconds passed' + % (percent, progress_size / (1024 * 1024), speed, duration)) + sys.stdout.flush() + +urllib.urlretrieve('$src', '$dst', reporthook=reporthook) +print('\n') +EOF + true + else + fail "No way to download $src" + fi +} + +export -f __download + +function __download_recipe_build() { + set -e + local recipe=$1 + local version=${2:-latest} + local do_checksum=${3:-true} + local do_checksign=${4:-false} + local do_cache=${5:-false} + local builds_url=${6:-http://kameleon.imag.fr/builds} + local dest_dir="${7:-$recipe}" + local dest="" + mkdir -p $dest_dir + pushd $dest_dir > /dev/null + echo "Downloading $recipe ($version):" + __download $builds_url/${recipe}_$version.manifest + if [ "$do_checksign" == "true" ]; then + __download $builds_url/${recipe}_$version.manifest.sign + gpg --verify ${recipe}_$version.manifest{.sign,} || fail "Cannot verify signature" + fi + for f in $(< ${recipe}_$version.manifest); do + if [[ $f =~ ^$recipe-cache_ ]] && [ "$do_cache" != "true" ]; then + continue + fi + if [[ $f =~ \.sha[[:digit:]]+sum$ ]]; then + if [ "$do_checksum" == "true" ]; then + __download $builds_url/$f + ${f##*.} -c $f || fail "Cannot verify checksum" + if [ "$do_checksign" == "true" ]; then + __download $builds_url/$f.sign + gpg --verify $f{.sign,} || fail "Cannot verify signature" + fi + fi + else + __download $builds_url/$f + echo -n "Link to version-less filename: " + dest=${f%_*}.tar.${f#*.tar.} + ln -fv $f $dest + fi + done + popd > /dev/null + export UPSTREAM_TARBALL="$dest_dir/$dest" + set +e +} + +export -f __download_recipe_build + +function __download_kadeploy_environment_image() { + set -e + local kaenv_name=$1 + local kaenv_user=$2 + local kaenv_version=$3 + local remote=$4 + local dest_dir=${5:-$kaenv_name} + mkdir -p $dest_dir + echo "Retrieve image for Kadeploy environment $kaenv_name" + ${remote:+ssh $remote }which kaenv3 > /dev/null || fail "kaenv3 command not found (${remote:-localhost})" + # retrieve image[file], image[kind] and image[compression] from kaenv3 + declare -A image + __kaenv() { local k=${2%%:*}; image[$k]=${2#*:}; } + mapfile -s 1 -t -c1 -C __kaenv < <(${remote:+ssh $remote }kaenv3${kaenv_user:+ -u $kaenv_user}${kaenv_version:+ --env-version $kaenv_version} -p $kaenv_name | grep -A3 -e '^image:' | sed -e 's/ //g') + [ -n "${image[file]}" ] || fail "Failed to retrieve environment $kaenv_name" + if [ "${image[compression]}" == "gzip" ]; then + image[compression]="gz" + elif [ "${image[compression]}" == "bzip2" ]; then + image[compression]="bz2" + fi + image[protocol]=${image[file]%%:*} + image[path]=${image[file]#*://} + image[filename]=${image[path]##*/} + local dest=$dest_dir/${image[filename]%%.*}.${image[kind]}.${image[compression]} + if [ "${image[kind]}" == "tar" ]; then + if [ "${image[protocol]}" == "http" -o "${image[protocol]}" == "https" ]; then + __download ${image[file]} $dest + else + if [ "${image[protocol]}" == "server" ]; then + # If server:// => see if available locally (NFS) or fail, same as if local:// <=> "" + echo "Image is server side, try and fetch it from local file ${image[path]}" + fi + [ -r ${image[path]} ] || fail "Cannot retrieve ${image[file]}" + cp -v ${image[path]} $dest + fi + else # dd or whatever + fail "Image format${image[kind]:+ ${image[kind]}} is not supported" + fi + export UPSTREAM_TARBALL=$dest + set +e +} + +export -f __download_kadeploy_environment_image + +function __find_linux_boot_device() { + local PDEVICE=`stat -c %04D /boot` + for file in $(find /dev -type b 2>/dev/null) ; do + local CURRENT_DEVICE=$(stat -c "%02t%02T" $file) + if [ $CURRENT_DEVICE = $PDEVICE ]; then + ROOTDEVICE="$file" + break; + fi + done + echo "$ROOTDEVICE" +} + +export -f __find_linux_boot_device + + +function __find_free_port() { + local begin_port=$1 + local end_port=$2 + + local port=$begin_port + local ret=$(nc -z 127.0.0.1 $port && echo in use || echo free) + while [ $port -le $end_port ] && [ "$ret" == "in use" ] + do + local port=$[$port+1] + local ret=$(nc -z 127.0.0.1 $port && echo in use || echo free) + done + + # manage loop exits + if [[ $port -gt $end_port ]] + then + fail "No free port available between $begin_port and $end_port" + fi + + echo $port +} + +export -f __find_free_port diff --git a/default/steps/export/save_appliance_VM.yaml b/default/steps/export/save_appliance_VM.yaml new file mode 100644 index 0000000..b064d02 --- /dev/null +++ b/default/steps/export/save_appliance_VM.yaml @@ -0,0 +1,23 @@ +# +# Save Appliance from virtual machine +# +- export_appliance_script: $${kameleon_data_dir}/helpers/export_appliance.py + +# Zero free unallocated blocks from ext2/3 file-systems before export to +# reduce image size +- zerofree: true + +- save_appliance: + - check_cmd_local: python2 + - exec_local: | + if [ "$${zerofree}" = "true" ]; then + EXPORT_OPTS="--zerofree" + else + EXPORT_OPTS="" + fi + - exec_local: | + python2 $${export_appliance_script} $${image_disk}.$${image_format} \ + -o $${appliance_filename} \ + --formats $${appliance_formats} \ + --tar-compression-level $${appliance_tar_compression_level} \ + --tar-excludes $${appliance_tar_excludes} $EXPORT_OPTS diff --git a/default/steps/setup/debian/clean_system.yaml b/default/steps/setup/debian/clean_system.yaml new file mode 100644 index 0000000..399c339 --- /dev/null +++ b/default/steps/setup/debian/clean_system.yaml @@ -0,0 +1,34 @@ +- enable_lighten: false + +- clean_user: + - on_setup_clean: + - exec_in: | + if id kameleon > /dev/null 2>&1; then + echo "Removing the kameleon user" + userdel -r kameleon 2> >(grep -v "userdel: kameleon mail spool (/var/mail/kameleon) not found" ) + fi + +- clean_apt: + - on_setup_clean: + - apt-get_in: autoremove + - apt-get_in: autoclean + - apt-get_in: purge + - apt-get_in: clean + - exec_in: | + if [ $${enable_lighten} = true ]; then + rm -rf /var/lib/apt/lists/* + rm -rf /usr/share/locale/* + rm -rf /usr/share/man/* + rm -rf /usr/share/doc/* + fi + +- clean_network: + - on_setup_clean: + - exec_in: rm -rf /var/lib/dhcp/* + +- clean_udev: + - on_setup_clean: + - exec_in: rm -rf /etc/udev/rules.d/70-persistent-net.rules + - exec_in: rm -rf /dev/.udev/ + - exec_in: touch /etc/udev/rules.d/70-persistent-net.rules + - exec_in: rm -rf /lib/udev/rules.d/75-persistent-net-generator.rules
\ No newline at end of file diff --git a/default/steps/setup/debian/minimal_install.yaml b/default/steps/setup/debian/minimal_install.yaml new file mode 100644 index 0000000..d1cdc69 --- /dev/null +++ b/default/steps/setup/debian/minimal_install.yaml @@ -0,0 +1,6 @@ + +- set_root_password: + - exec_in: echo -n 'root:$${root_password}' | chpasswd + +- upgrade_system: + - apt-get_in: dist-upgrade |