Revision as of 22:23, 2 August 2010

Preparing Host

Starting with a minimal Suse 11.2 install

Define some base data

ROOTFS=/data/11.2min/image-root
ROUTER_IP=10.0.0.1
VG=VM

Based on http://en.opensuse.org/Build_Service/KIWI/Cookbook

zypper ar http://download.opensuse.org/repositories/Virtualization:/Appliances/openSUSE_11.2/ Virtualization:Appliances
zypper refresh

zypper in kiwi kiwi-templates kiwi-desc-xenboot squashfs emacs

Prepare the storage for LV usage

parted /ddev/sdb
 mklabel
 gpt
 mkpart p1 0 10%
 mkpart p2 10% 20%
 mkpart p3 20% 30%
 mkpart p4 30% 40%
 mkpart p5 40% 50%
 mkpart p6 50% 60%
 mkpart p7 60% 70%
 mkpart p8 70% 80%
 mkpart p9 80% 90%
 mkpart p10 90% 100%
 quit

Then make the VG

pvcreate /dev/sdb?*
vgcreate $VG /dev/sdb1

Prepare an openSUSE minimal image:

mkdir -p /data/11.2min
rm -rf /data/11.2min/image-root
kiwi --prepare suse-11.2-JeOS --root $ROOTFS --add-profile xenFlavour --add-package less --add-package iputils --add-package kernel-xen --add-package wget --add-package less --add-package iputils --add-package terminfo --add-package emacs --add-package sudo

Update the config & modules:

echo default $ROUTER_IP > $ROOTFS/etc/sysconfig/network/routes
echo NETCONFIG_DNS_POLICY=\"\" >> $ROOTFS/etc/sysconfig/network/config
echo nameserver 8.8.8.8 > $ROOTFS/etc/resolv.conf
echo default $ROUTER_IP > $ROOTFS/etc/sysconfig/network/routes
cat << EOF >$ROOTFS/etc/sysconfig/network/ifcfg-eth0
BOOTPROTO='static'
BROADCAST=''
STARTMODE='onboot'
EOF
echo /dev/xvda1 swap swap defaults 0 0 >> $ROOTFS/etc/fstab

Prepare some overlay data from the main host to allow ssh into guests etc

# Allow user ssh to all VMs and retain sudo rights
# Should probably be done periodically somehow
mkdir /data/vm_overlay/home
cp -ar /home /data/vm_overlay/home
mkdir -p /data/vm_overlay/etc/sysconfig/
ln /etc/passwd /data/vm_overlay/etc/
ln /etc/shadow /data/vm_overlay/etc/
ln /etc/group /data/vm_overlay/etc/
ln /etc/sudoers /data/vm_overlay/etc/
# Fix for screen/bash ctrl-arrow
ln /etc/inputrc /data/vm_overlay/etc/
# Network proxy information
ln /etc/sysconfig/proxy /data/vm_overlay/etc/sysconfig/

(note JeOS 11.2 won't allow su - <user> or ssh when user is disabled using ! in /etc/shadow. Instead use an impossible hash.)

Useful

chroot_vm() {
GUEST=$1
xm list | grep "^$GUEST " && echo "$GUEST is running" && return
mkdir /mnt/${GUEST}_chroot/
mount /dev/$VG/${GUEST}_root /mnt/${GUEST}_chroot/ &&
chroot /mnt/${GUEST}_chroot/
umount /mnt/${GUEST}_chroot/
rmdir /mnt/${GUEST}_chroot/
}

Make VMs - per VM

Make sure you setup the base data environment

VG=VM
FE_IP=10.0.0.10
BE_IP=10.0.0.11

Create Xen volumes

mk_lv() {
GUEST=$1
lvremove /dev/$VG/${GUEST}_*
lvcreate -L 10G $VG -n ${GUEST}_root
lvcreate -L 2G  $VG -n ${GUEST}_swap
mkswap -f /dev/$VG/${GUEST}_swap
}

For cfe & cbe

mk_lv cfe
mk_lv cbe

Copy the minimal image and overlay to the VM root disk and set an IP

mk_fs() {
GUEST=$1
IP=$2
mkdir /mnt/lvm
echo mkfs &&
mkfs -text4 /dev/$VG/${GUEST}_root &&
echo mounting &&
mount /dev/$VG/${GUEST}_root /mnt/lvm &&
echo copy rootfs &&
rsync -HAXa /data/11.2min/image-root/ /mnt/lvm/ &&
echo copy overlay &&
echo ${GUEST}.meego.com > /mnt/lvm/etc/HOSTNAME &&
echo "IPADDR='$IP/24'" >> /mnt/lvm/etc/sysconfig/network/ifcfg-eth0 &&
rsync -HAXa /data/vm_overlay/ /mnt/lvm/ &&
echo sync &&
sync &&
echo umount &&
umount /mnt/lvm
}

For cfe & cbe

mk_fs cfe $IP_FE
mk_fs cbe $IP_BE

Make per-machine files in /etc/xen/ with unique MACs Additional LV space can be allocated here too

mk_g() {
GUEST=$1
MAC=$2
cat <<EOF > /etc/xen/$GUEST.cfg
name='${GUEST}'
disk=['phy:/dev/$VG/${GUEST}_root,xvda2,w', 'phy:/dev/$VG/${GUEST}_swap,xvda1,w']
vif=['mac=$MAC, bridge=eth0']
memory='2048'

root='/dev/xvda2 rw'
kernel='/boot/vmlinuz-2.6.31.12-0.2-xen'
ramdisk='/boot/initrd-2.6.31.12-0.2-xen'
extra='clocksource=jiffies console=hvc0 xencons=tty'

on_poweroff='destroy'
on_reboot='restart'
on_crash='restart'
EOF
}

For cfe & cbe

mk_g cfe 00:16:3E:40:B5:FE
mk_g cbe 00:16:3E:40:B5:BE

Then start the VMs:

xm create /etc/xen/cfe.cfg
xm create /etc/xen/cbe.cfg

Installing the Backend

On this guest we need also to setup openSUSE Tools repository:

cd /etc/zypp/repos.d/;
wget http://download.opensuse.org/repositories/Maemo:/MeeGo-Infra:/OBS/Tools_Unstable_openSUSE_11.2/Maemo:MeeGo-Infra:OBS.repo
wget http://download.opensuse.org/repositories/openSUSE:/Tools:/Unstable/openSUSE_11.2/openSUSE:Tools:Unstable.repo
wget http://download.opensuse.org/repositories/openSUSE:/Tools/openSUSE_11.2/openSUSE:Tools.repo
zypper ref
# Accept the trust key
<pre>
Install:
<pre>
zypper in obs-server obs-source_service obs-signer obs-utils createrepo dpkg lighttpd

nano /etc/sysconfig/obs-server
OBS_SCHEDULER_ARCHITECTURES="i586 x86_64 armv5el armv7el"

Now /usr/lib/obs/server/BSConfig.pm needs to point to correct server names corresponding to source server, where workers are going to download the source, and the repository server, where RPM repos are going to be shared to users.

What's needed here?????????

nano /usr/lib/obs/server/BSConfig.pm
#add
$hostname="cbuild.meego.com";

our $srcserver = "http://$hostname:5352";
our $reposerver = "http://$hostname:5252";
our $serviceserver = "http://$hostname:5152";
our $servicedir = "/usr/lib/obs/service/";
#

Configure services as daemons

chkconfig --add obsrepserver obssrcserver obsscheduler obsdispatcher obspublisher obswarden obssigner

Start Services

rcobsrepserver start
rcobssrcserver start
rcobsscheduler start
rcobsdispatcher start
rcobspublisher start
rcobswarden start
rcobssigner start

Not started

rcobsservice

Lighttpd

lighttpd also needs to be available on backend server. This is required to provide directory listing on the repositories available on this server when an http/s request to maemo-repo is made through web ui.

Create a new file under /etc/lighttpd/vhosts.d/. It can be obs.conf as well, and add:

nano /etc/lighttpd/vhosts.d/obs.conf

$HTTP["host"] =~ "crepo.meego.com" {
  server.name = "crepo.meego.com"

  server.document-root = "/srv/obs/repos/"
  dir-listing.activate = "enable"
}

To enable vhosts, remember to uncomment the following in the 'custom includes':

nano /etc/lighttpd/lighttpd.conf
##
  ## custom includes like vhosts.
  ##
  #include "conf.d/config.conf"
  # following line uncommented as per
  # /usr/share/doc/packages/obs-api/README.SETUP
  include_shell "cat vhosts.d/*.conf"

And disable ipv6 unless it's secured correctly

server.use-ipv6 = "disable"

Start lighttpd

#first add it as deamon
chkconfig --add lighttpd
rclighttpd start

Installing the Frontend

On this guest we need also to setup openSUSE Tools repository:

cd /etc/zypp/repos.d/;
wget http://download.opensuse.org/repositories/Maemo:/MeeGo-Infra:/OBS/Tools_Unstable_openSUSE_11.2/Maemo:MeeGo-Infra:OBS.repo
wget http://download.opensuse.org/repositories/openSUSE:/Tools:/Unstable/openSUSE_11.2/openSUSE:Tools:Unstable.repo
wget http://download.opensuse.org/repositories/openSUSE:/Tools/openSUSE_11.2/openSUSE:Tools.repo
zypper ref
# Accept the trust key
<pre>


Install obs-api (It's going to install lighttpd webserver by dependency for you).
<pre>
zypper in obs-api memcached

Setup MySQL

MySQL server needs to be installed and configured to start as daemon

chkconfig --add mysql
rcmysql start

Setup a secure installation, if it's the first time starting MySQL

/usr/bin/mysql_secure_installation

touch /root/.my.cnf
chmod 0600 /root/.my.cnf
nano /root/.my.cnf

[client]
user = root
password = <PASSWORD>
[mysqladmin]
user= root
password = <PASSWORD>

The frontend instance holds 2 applications, the API and the webui. Each one need a database created

mysql -u root -p
 create database api_production;
 create database webui_production;

Add obs user to handle these databases

GRANT all privileges
      ON api_production.*
      TO 'obs'@'%', 'obs'@'localhost' IDENTIFIED BY '************';
GRANT all privileges
      ON webui_production.*
      TO 'obs'@'%', 'obs'@'localhost' IDENTIFIED BY '************';
FLUSH PRIVILEGES;

Now secure the passwd storing config files

touch /srv/www/obs/api/config/database.yml
touch /srv/www/obs/webui/config/database.yml
chmod 600 /srv/www/obs/api/config/database.yml
chmod 600 /srv/www/obs/webui/config/database.yml
chown lighttpd /srv/www/obs/api/config/database.yml
chown lighttpd /srv/www/obs/webui/config/database.yml

Configure your MySQL user and password in the "production:" section of the API config:

nano /srv/www/obs/api/config/database.yml
#change the production section
production:
  adapter: mysql
  database: api_production
  username: obs
  password: ************

Do the same for the webui. It's configured, by default to use SQLite, but since we're configuring the cluster for production environment, let's bind it to mysql:

nano /srv/www/obs/webui/config/database.yml
#change the production section
production:
  adapter: mysql
  database: webui_production
  username: obs
  password: ************

Populate the database

mkdir -p /srv/www/obs/api/db/data/production
cd /srv/www/obs/api/
RAILS_ENV="production" rake db:migrate
chown lighttpd.lighttpd log/*

cd /srv/www/obs/webui/
RAILS_ENV="production" rake db:migrate
chown lighttpd.lighttpd log/*

You can check the migration was successful verifying the “migrated” message at the end of each statement.

Setup and configure lighttpd for the API and webui

You need to setup the correct hostnames to where webui, API and repo server are going to point to

Edit /etc/lighttpd/vhosts.d/obs.conf

$HTTP["host"] =~ "^cbuild" {
  rails_app   = "webui"
  rails_root  = "/srv/www/obs/webui"
  rails_procs = 10
  # production/development are typical values here
  rails_mode  = "production"

  log_root = "/srv/www/obs/webui/log"

  include "vhosts.d/rails.inc"
}
$HTTP["host"] =~ "^capi" {
  rails_app   = "api"
  rails_root  = "/srv/www/obs/api"
  rails_procs = 3
  # production/development are typical values here
  rails_mode  = "production"

  log_root = "/srv/www/obs/api/log"

  include "vhosts.d/rails.inc"
}
$HTTP["host"] =~ "download" {
# This should point to an rsync populated download repo
#  server.name = "download.obs.maemo.org"
#  server.document-root = "/srv/obs/repos/"

  proxy.server = ( "" => ( (
        "host" => "10.1.1.11",
        "port" => 80
      ))
  )
}

To enable these vhosts, make sure to uncomment the following in the 'custom includes' section at the bottom of /etc/lighttpd/lighttpd.conf:

nano /etc/lighttpd/lighttpd.conf
##
## custom includes like vhosts.
##
#include "conf.d/config.conf"
include_shell "cat /etc/lighttpd/vhosts.d/*.conf"

Also need to disable IPv6 unkess it's secured

server.use-ipv6 = "disable"

Also, the modules "mod_magnet", "mod_rewrite" and FastCGI need to be enabled by uncommenting the corresponding lines in /etc/lighttpd/modules.conf:

server.modules = (
  "mod_access",
#  "mod_alias",
#  "mod_auth",
#  "mod_evasive",
#  "mod_redirect",
  "mod_rewrite",
#  "mod_setenv",
#  "mod_usertrack",
)

##
## mod_magnet
##
include "conf.d/magnet.conf"

##
## FastCGI (mod_fastcgi)
##
include "conf.d/fastcgi.conf"

You need also to configure /srv/www/obs/webui/config/environments/production.rb to point to correct server names:

nano /srv/www/obs/webui/config/environments/production.rb
FRONTEND_HOST = "capi.meego.com"
FRONTEND_PORT = 80
EXTERNAL_FRONTEND_HOST = "capi.meego.com"
BUGZILLA_HOST = "http://bugs.moego.com/"
DOWNLOAD_URL = "http://cdownload.meego.com/"

Do the same for /srv/www/obs/api/config/environments/production.rb. As soon your backend is not on the same machine as the api (frontend), change the following:

nano /srv/www/obs/api/config/environments/production.rb
SOURCE_HOST = "cbe.meego.com"
SOURCE_PORT = 5352
DOWNLOAD_URL='http://cdownload.meego.com/'

ligthttpd user and group need to be the owner of api and webui dirs (as well as log and tmp):

chown -R lighttpd.lighttpd /srv/www/obs/{api,webui}

Make sure TCP port 5352 is open on the firewall. Ensure lighttpd and obs ui helpers start:

chkconfig --add memcached
chkconfig --add lighttpd
chkconfig --add obsapidelayed
chkconfig --add obswebuidelayed

rcmemcached start
rclighttpd start
rcobsapidelayed start
rcobswebuidelayed start

rcobsapidelayed

Preparing Worker Host

vgadd OBS /dev/sda4
vgcreate OBS /dev/sda4

Installing the Workers

The other hosts on the cluster are reserved to be used as workers, where package builds are going to place.

The same openSUSE Tools repository addition must be done for each worker.

cd /etc/zypp/repos.d/;
wget http://download.opensuse.org/repositories/Maemo:/MeeGo-Infra:/OBS/Tools_Unstable_openSUSE_11.2/Maemo:MeeGo-Infra:OBS.repo
wget http://download.opensuse.org/repositories/openSUSE:/Tools:/Unstable/openSUSE_11.2/openSUSE:Tools:Unstable.repo
wget http://download.opensuse.org/repositories/openSUSE:/Tools/openSUSE_11.2/openSUSE:Tools.repo
zypper ref
# Accept the trust key

</pre> zypper in obs-worker quemu mount-static bash-static </pre>

(mount-static and bash-static are needed on the worker for rpm cross-compile to work)

For Xen workers we need a suitable initrd:

mkinitrd -d /dev/null -m "ext4 binfmt_misc" -k vmlinuz-2.6.31.12-0.2-xen -i initrd-2.6.31.12-0.2-xen-obs_worker

This will create an initrd for your kernel

Kernel image:   /boot/vmlinuz-2.6.31.12-0.2-xen
Initrd image:   /boot/initrd-2.6.31.12-0.2-xen-obs_worker

This assumes you have a VG dedicated to workers called "OBS"

Edit the file /etc/sysconfig/obs-worker in order to point to correct repository server.

</pre> nano /etc/sysconfig/obs-worker OBS_SRC_SERVER="cbe:5352" OBS_REPO_SERVERS="cbe:5252" OBS_VM_TYPE="xen" OBS_VM_KERNEL="/boot/vmlinuz-2.6.31.12-0.2-xen" OBS_VM_INITRD="/boot/initrd-2.6.31.12-0.2-xen-obs_worker" OBS_VM_DISK_AUTOSETUP_ROOT_FILESIZE="8192" OBS_VM_DISK_AUTOSETUP_SWAP_FILESIZE="2048" OBS_INSTANCE_MEMORY="1024" OBS_STORAGE_AUTOSETUP="yes" OBS_SETUP_WORKER_PARTITIONS="use_obs_vg" OBS_WORKER_ROOT_SIZE="8192" OBS_WORKER_SWAP_SIZE="2048" </pre>

The obsstoragesetup will wipe the OBS VG and create root/swap LVs for each worker

rcobsstoragesetup start

Output: </pre> mdadm: No arrays found in config file or automatically Waiting for udev to settle... Scanning for LVM volume groups...

 Reading all physical volumes.  This may take a while...
 Found volume group "OBS" using metadata type lvm2

Activating LVM volume groups...

 0 logical volume(s) in volume group "OBS" now active
                                                                                                            done
 Logical volume "worker_root_1" created
 Logical volume "worker_swap_1" created
 Logical volume "worker_root_2" created
 Logical volume "worker_swap_2" created
 Logical volume "worker_root_3" created
 Logical volume "worker_swap_3" created
 Logical volume "worker_root_4" created
 Logical volume "worker_swap_4" created
 Logical volume "worker_root_5" created
 Logical volume "worker_swap_5" created
 Logical volume "worker_root_6" created
 Logical volume "worker_swap_6" created
 Logical volume "worker_root_7" created
 Logical volume "worker_swap_7" created
 Logical volume "worker_root_8" created
 Logical volume "worker_swap_8" created
 Logical volume "worker_root_9" created
 Logical volume "worker_swap_9" created
 Logical volume "worker_root_10" created
 Logical volume "worker_swap_10" created
 Logical volume "worker_root_11" created
 Logical volume "worker_swap_11" created
 Logical volume "worker_root_12" created
 Logical volume "worker_swap_12" created
 Logical volume "worker_root_13" created
 Logical volume "worker_swap_13" created
 Logical volume "worker_root_14" created
 Logical volume "worker_swap_14" created
 Logical volume "worker_root_15" created
 Logical volume "worker_swap_15" created
 Logical volume "worker_root_16" created
 Logical volume "worker_swap_16" created
 Logical volume "cache" created

mke2fs 1.41.9 (22-Aug-2009) Filesystem label= OS type: Linux Block size=4096 (log=2) Fragment size=4096 (log=2) 3555328 inodes, 14201856 blocks 710092 blocks (5.00%) reserved for the super user First data block=0 Maximum filesystem blocks=4294967296 434 block groups 32768 blocks per group, 32768 fragments per group 8192 inodes per group Superblock backups stored on blocks:

       32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208,
       4096000, 7962624, 11239424

Writing inode tables: done Creating journal (32768 blocks): done Writing superblocks and filesystem accounting information: done

This filesystem will be automatically checked every 39 mounts or 180 days, whichever comes first. Use tune2fs -c or -i to override. Looking for OBS Server LVM Volume Setup local storage Looking for OBS Worker Cache LVM Volume Setting up OBS Workers according to LVM Volumes Found XEN virtualization

                                                                                                            done

</pre>

Test

xm create -c /var/run/obs/worker/8/build/xen.conf name=build:root8 memory=40 disk=phy:/dev/mapper/OBS-worker_root8,hda1,w disk=phy:/dev/mapper/OBS-worker_swap8,hda2,w extra="init=/.build/initscript_qemu_vm panic=1 console=ttyS0

Start the worker service:

chkconfig --add obsworker
rcobsworker start

for i in 1 2 3 4 5 6 7 8 9 10

   do echo 1024 > /OBS.worker/root$i/memory

done for i in 1 2 3 4 5 6 7 8 9 10 do

   lvcreate -L 4G -n worker_root$i OBS
   mkfs -text4 /dev/OBS/worker_root$i
   lvcreate -L 1G -n worker_swap$i OBS
   mkswap /dev/OBS/worker_swap$i

done for i in 1 2 3 4 5 6 7 8 9 10 do

 mkswap -f /dev/OBS/worker_swap$i

done

lvremove /dev/OBS/worker_root9 cat /etc/sysconfig/bootloader

mount /dev/OBS/cache

rcobsworker start