(→Preparing Host) |
(more VMs) |
||
| Line 120: | Line 120: | ||
} | } | ||
</pre> | </pre> | ||
| - | + | On the appropriate xen host: | |
<pre> | <pre> | ||
mk_lv cfe | mk_lv cfe | ||
mk_lv cbe | mk_lv cbe | ||
| + | mk_lv cstore | ||
| + | mk_lv csign | ||
</pre> | </pre> | ||
| Line 148: | Line 150: | ||
} | } | ||
</pre> | </pre> | ||
| - | + | On the appropriate xen host: | |
<pre> | <pre> | ||
mk_fs cfe $IP_FE | mk_fs cfe $IP_FE | ||
mk_fs cbe $IP_BE | mk_fs cbe $IP_BE | ||
| + | mk_fs cstore $IP_STORE | ||
| + | mk_fs csign $IP_STORE | ||
</pre> | </pre> | ||
| Line 177: | Line 181: | ||
} | } | ||
</pre> | </pre> | ||
| - | + | On the appropriate xen host: | |
<pre> | <pre> | ||
mk_g cfe 00:16:3E:40:B5:FE | mk_g cfe 00:16:3E:40:B5:FE | ||
mk_g cbe 00:16:3E:40:B5:BE | mk_g cbe 00:16:3E:40:B5:BE | ||
| + | mk_g cstore 00:16:3E:40:B5:5E | ||
| + | mk_g csign 00:16:3E:40:51:64 | ||
</pre> | </pre> | ||
Contents |
Starting with a minimal Suse 11.2 install
Define some base data
ROOTFS=/data/11.2min/image-root ROUTER_IP=10.0.0.1 VG=VM
Based on http://en.opensuse.org/Build_Service/KIWI/Cookbook
zypper ar http://download.opensuse.org/repositories/Virtualization:/Appliances/openSUSE_11.2/ Virtualization:Appliances zypper refresh
zypper in kiwi kiwi-templates kiwi-desc-xenboot squashfs emacs
Prepare the storage for LV usage
parted /dev/sdb mklabel gpt yes mkpart p1 0 10% mkpart p2 10% 20% mkpart p3 20% 30% mkpart p4 30% 40% mkpart p5 40% 50% mkpart p6 50% 60% mkpart p7 60% 70% mkpart p8 70% 80% mkpart p9 80% 90% mkpart p10 90% 100% quit
Then make the VG
pvcreate /dev/sdb?* vgcreate $VG /dev/sdb1
Prepare an openSUSE minimal image:
mkdir -p /data/11.2min rm -rf /data/11.2min/image-root kiwi --prepare suse-11.2-JeOS --root $ROOTFS --add-profile xenFlavour --add-package less --add-package iputils --add-package kernel-xen --add-package wget --add-package less --add-package iputils --add-package terminfo --add-package emacs --add-package sudo
Update the config & modules:
echo default $ROUTER_IP > $ROOTFS/etc/sysconfig/network/routes echo NETCONFIG_DNS_POLICY=\"\" >> $ROOTFS/etc/sysconfig/network/config echo nameserver 8.8.8.8 > $ROOTFS/etc/resolv.conf echo default $ROUTER_IP > $ROOTFS/etc/sysconfig/network/routes cat << EOF >$ROOTFS/etc/sysconfig/network/ifcfg-eth0 BOOTPROTO='static' BROADCAST='' STARTMODE='onboot' EOF echo /dev/xvda1 swap swap defaults 0 0 >> $ROOTFS/etc/fstab
Prepare some overlay data from the main host to allow ssh into guests
etc
# Allow user ssh to all VMs and retain sudo rights # Should probably be done periodically somehow mkdir /data/vm_overlay/home cp -ar /home /data/vm_overlay/home mkdir -p /data/vm_overlay/etc/sysconfig/ ln /etc/passwd /data/vm_overlay/etc/ ln /etc/shadow /data/vm_overlay/etc/ ln /etc/group /data/vm_overlay/etc/ ln /etc/sudoers /data/vm_overlay/etc/ # Fix for screen/bash ctrl-arrow ln /etc/inputrc /data/vm_overlay/etc/ # Network proxy information ln /etc/sysconfig/proxy /data/vm_overlay/etc/sysconfig/
(note JeOS 11.2 won't allow su - <user> or ssh when user is disabled using ! in /etc/shadow. Instead use an impossible hash.)
chroot_vm() {
GUEST=$1
xm list | grep "^$GUEST " && echo "$GUEST is running" && return
mkdir /mnt/${GUEST}_chroot/
mount /dev/$VG/${GUEST}_root /mnt/${GUEST}_chroot/ &&
chroot /mnt/${GUEST}_chroot/
umount /mnt/${GUEST}_chroot/
rmdir /mnt/${GUEST}_chroot/
}
Make sure you setup the base data environment
VG=VM FE_IP=10.0.0.10 BE_IP=10.0.0.11
Create Xen volumes
mk_lv() {
GUEST=$1
lvremove /dev/$VG/${GUEST}_*
lvcreate -L 10G $VG -n ${GUEST}_root
lvcreate -L 2G $VG -n ${GUEST}_swap
mkswap -f /dev/$VG/${GUEST}_swap
}
On the appropriate xen host:
mk_lv cfe mk_lv cbe mk_lv cstore mk_lv csign
Copy the minimal image and overlay to the VM root disk and set an IP
mk_fs() {
GUEST=$1
IP=$2
mkdir /mnt/lvm
echo mkfs &&
mkfs -text4 /dev/$VG/${GUEST}_root &&
echo mounting &&
mount /dev/$VG/${GUEST}_root /mnt/lvm &&
echo copy rootfs &&
rsync -HAXa /data/11.2min/image-root/ /mnt/lvm/ &&
echo copy overlay &&
echo ${GUEST}.meego.com > /mnt/lvm/etc/HOSTNAME &&
echo "IPADDR='$IP/24'" >> /mnt/lvm/etc/sysconfig/network/ifcfg-eth0 &&
rsync -HAXa /data/vm_overlay/ /mnt/lvm/ &&
echo sync &&
sync &&
echo umount &&
umount /mnt/lvm
}
On the appropriate xen host:
mk_fs cfe $IP_FE mk_fs cbe $IP_BE mk_fs cstore $IP_STORE mk_fs csign $IP_STORE
Make per-machine files in /etc/xen/ with unique MACs Additional LV space can be allocated here too
mk_g() {
GUEST=$1
MAC=$2
cat <<EOF > /etc/xen/$GUEST.cfg
name='${GUEST}'
disk=['phy:/dev/$VG/${GUEST}_root,xvda2,w', 'phy:/dev/$VG/${GUEST}_swap,xvda1,w']
vif=['mac=$MAC, bridge=eth0']
memory='2048'
root='/dev/xvda2 rw'
kernel='/boot/vmlinuz-2.6.31.12-0.2-xen'
ramdisk='/boot/initrd-2.6.31.12-0.2-xen'
extra='clocksource=jiffies console=hvc0 xencons=tty'
on_poweroff='destroy'
on_reboot='restart'
on_crash='restart'
EOF
}
On the appropriate xen host:
mk_g cfe 00:16:3E:40:B5:FE mk_g cbe 00:16:3E:40:B5:BE mk_g cstore 00:16:3E:40:B5:5E mk_g csign 00:16:3E:40:51:64
Then start the VMs:
xm create /etc/xen/cfe.cfg xm create /etc/xen/cbe.cfg
On this guest we need also to setup openSUSE Tools repository:
cd /etc/zypp/repos.d/; wget http://download.opensuse.org/repositories/Maemo:/MeeGo-Infra:/OBS/Tools_Unstable_openSUSE_11.2/Maemo:MeeGo-Infra:OBS.repo wget http://download.opensuse.org/repositories/openSUSE:/Tools:/Unstable/openSUSE_11.2/openSUSE:Tools:Unstable.repo wget http://download.opensuse.org/repositories/openSUSE:/Tools/openSUSE_11.2/openSUSE:Tools.repo zypper ref # Accept the trust key <pre> Install: <pre> zypper in obs-server obs-source_service obs-signer obs-utils createrepo dpkg lighttpd
nano /etc/sysconfig/obs-server OBS_SCHEDULER_ARCHITECTURES="i586 x86_64 armv5el armv7el"
Now /usr/lib/obs/server/BSConfig.pm needs to point to correct server names corresponding to source server, where workers are going to download the source, and the repository server, where RPM repos are going to be shared to users.
What's needed here?????????
nano /usr/lib/obs/server/BSConfig.pm #add $hostname="cbuild.meego.com"; our $srcserver = "http://$hostname:5352"; our $reposerver = "http://$hostname:5252"; our $serviceserver = "http://$hostname:5152"; our $servicedir = "/usr/lib/obs/service/"; #
Configure services as daemons
chkconfig --add obsrepserver obssrcserver obsscheduler obsdispatcher obspublisher obswarden obssigner
Start Services
rcobsrepserver start rcobssrcserver start rcobsscheduler start rcobsdispatcher start rcobspublisher start rcobswarden start rcobssigner start
Not started
rcobsservice
lighttpd also needs to be available on backend server. This is required to provide directory listing on the repositories available on this server when an http/s request to maemo-repo is made through web ui.
Create a new file under /etc/lighttpd/vhosts.d/. It can be obs.conf as well, and add:
nano /etc/lighttpd/vhosts.d/obs.conf
$HTTP["host"] =~ "crepo.meego.com" {
server.name = "crepo.meego.com"
server.document-root = "/srv/obs/repos/"
dir-listing.activate = "enable"
}
To enable vhosts, remember to uncomment the following in the 'custom includes':
nano /etc/lighttpd/lighttpd.conf ## ## custom includes like vhosts. ## #include "conf.d/config.conf" # following line uncommented as per # /usr/share/doc/packages/obs-api/README.SETUP include_shell "cat vhosts.d/*.conf"
And disable ipv6 unless it's secured correctly
server.use-ipv6 = "disable"
Start lighttpd
#first add it as deamon chkconfig --add lighttpd rclighttpd start
On this guest we need also to setup openSUSE Tools repository:
cd /etc/zypp/repos.d/; wget http://download.opensuse.org/repositories/Maemo:/MeeGo-Infra:/OBS/Tools_Unstable_openSUSE_11.2/Maemo:MeeGo-Infra:OBS.repo wget http://download.opensuse.org/repositories/openSUSE:/Tools:/Unstable/openSUSE_11.2/openSUSE:Tools:Unstable.repo wget http://download.opensuse.org/repositories/openSUSE:/Tools/openSUSE_11.2/openSUSE:Tools.repo zypper ref # Accept the trust key <pre> Install obs-api (It's going to install lighttpd webserver by dependency for you). <pre> zypper in obs-api memcached
MySQL server needs to be installed and configured to start as daemon
chkconfig --add mysql rcmysql start
Setup a secure installation, if it's the first time starting MySQL
/usr/bin/mysql_secure_installation
touch /root/.my.cnf chmod 0600 /root/.my.cnf nano /root/.my.cnf [client] user = root password = <PASSWORD> [mysqladmin] user= root password = <PASSWORD>
The frontend instance holds 2 applications, the API and the webui. Each one need a database created
mysql -u root -p create database api_production; create database webui_production;
Add obs user to handle these databases
GRANT all privileges
ON api_production.*
TO 'obs'@'%', 'obs'@'localhost' IDENTIFIED BY '************';
GRANT all privileges
ON webui_production.*
TO 'obs'@'%', 'obs'@'localhost' IDENTIFIED BY '************';
FLUSH PRIVILEGES;
Now secure the passwd storing config files
touch /srv/www/obs/api/config/database.yml touch /srv/www/obs/webui/config/database.yml chmod 600 /srv/www/obs/api/config/database.yml chmod 600 /srv/www/obs/webui/config/database.yml chown lighttpd /srv/www/obs/api/config/database.yml chown lighttpd /srv/www/obs/webui/config/database.yml
Configure your MySQL user and password in the "production:" section of the API config:
nano /srv/www/obs/api/config/database.yml #change the production section production: adapter: mysql database: api_production username: obs password: ************
Do the same for the webui. It's configured, by default to use SQLite, but since we're configuring the cluster for production environment, let's bind it to mysql:
nano /srv/www/obs/webui/config/database.yml #change the production section production: adapter: mysql database: webui_production username: obs password: ************
Populate the database
mkdir -p /srv/www/obs/api/db/data/production cd /srv/www/obs/api/ RAILS_ENV="production" rake db:migrate chown lighttpd.lighttpd log/* cd /srv/www/obs/webui/ RAILS_ENV="production" rake db:migrate chown lighttpd.lighttpd log/*
You can check the migration was successful verifying the “migrated” message at the end of each statement.
Setup and configure lighttpd for the API and webui
You need to setup the correct hostnames to where webui, API and repo server are going to point to
Edit /etc/lighttpd/vhosts.d/obs.conf
$SERVER["socket"] == "192.168.60.100:443" {
ssl.engine = "enable"
ssl.pemfile = "certificate.pem"
$HTTP["host"] =~ "^cbuild" {
server.name = "cbuild.meego.com"
rails_app = "webui"
rails_root = "/srv/www/obs/webui"
rails_procs = 10
# production/development are typical values here
rails_mode = "production"
log_root = "/srv/www/obs/webui/log"
include "vhosts.d/rails.inc"
}
$HTTP["host"] =~ "^capi" {
server.name = "capi.meego.com"
rails_app = "api"
rails_root = "/srv/www/obs/api"
rails_procs = 10
# production/development are typical values here
rails_mode = "production"
log_root = "/srv/www/obs/api/log"
include "vhosts.d/rails.inc"
}
}
$HTTP["host"] =~ "download" {
# This should point to an rsync populated download repo
# server.name = "download.obs.maemo.org"
# server.document-root = "/srv/obs/repos/"
proxy.server = ( "" => ( (
"host" => "10.1.1.11",
"port" => 80
))
)
}
To enable these vhosts, make sure to uncomment the following in the 'custom includes' section at the bottom of /etc/lighttpd/lighttpd.conf:
nano /etc/lighttpd/lighttpd.conf ## ## custom includes like vhosts. ## #include "conf.d/config.conf" include_shell "cat /etc/lighttpd/vhosts.d/*.conf"
Also need to disable IPv6 unkess it's secured
server.use-ipv6 = "disable"
Also, the modules "mod_magnet", "mod_rewrite" and FastCGI need to be enabled by uncommenting the corresponding lines in /etc/lighttpd/modules.conf:
server.modules = ( "mod_access", # "mod_alias", # "mod_auth", # "mod_evasive", # "mod_redirect", "mod_rewrite", # "mod_setenv", # "mod_usertrack", ) ## ## mod_magnet ## include "conf.d/magnet.conf" ## ## FastCGI (mod_fastcgi) ## include "conf.d/fastcgi.conf"
You need also to configure /srv/www/obs/webui/config/environments/production.rb to point to correct server names:
nano /srv/www/obs/webui/config/environments/production.rb FRONTEND_HOST = "capi.meego.com" FRONTEND_PORT = 80 EXTERNAL_FRONTEND_HOST = "capi.meego.com" BUGZILLA_HOST = "http://bugs.moego.com/" DOWNLOAD_URL = "http://cdownload.meego.com/"
Do the same for /srv/www/obs/api/config/environments/production.rb. As soon your backend is not on the same machine as the api (frontend), change the following:
nano /srv/www/obs/api/config/environments/production.rb SOURCE_HOST = "cbe.meego.com" SOURCE_PORT = 5352 DOWNLOAD_URL='http://cdownload.meego.com/'
ligthttpd user and group need to be the owner of api and webui dirs (as well as log and tmp):
chown -R lighttpd.lighttpd /srv/www/obs/{api,webui}
Make sure TCP port 5352 is open on the firewall. Ensure lighttpd and obs ui helpers start:
chkconfig --add memcached chkconfig --add lighttpd chkconfig --add obsapidelayed chkconfig --add obswebuidelayed rcmemcached start rclighttpd start rcobsapidelayed start rcobswebuidelayed start
rcobsapidelayed
vgadd OBS /dev/sda4 vgcreate OBS /dev/sda4
The other hosts on the cluster are reserved to be used as workers, where package builds are going to place.
The same openSUSE Tools repository addition must be done for each worker.
cd /etc/zypp/repos.d/; wget http://download.opensuse.org/repositories/Maemo:/MeeGo-Infra:/OBS/Tools_Unstable_openSUSE_11.2/Maemo:MeeGo-Infra:OBS.repo wget http://download.opensuse.org/repositories/openSUSE:/Tools:/Unstable/openSUSE_11.2/openSUSE:Tools:Unstable.repo wget http://download.opensuse.org/repositories/openSUSE:/Tools/openSUSE_11.2/openSUSE:Tools.repo zypper ref # Accept the trust key
zypper in obs-worker quemu mount-static bash-static
(mount-static and bash-static are needed on the worker for rpm cross-compile to work)
For Xen workers we need a suitable initrd:
mkinitrd -d /dev/null -m "ext4 binfmt_misc" -k vmlinuz-2.6.31.12-0.2-xen -i initrd-2.6.31.12-0.2-xen-obs_worker
This will create an initrd for your kernel
Kernel image: /boot/vmlinuz-2.6.31.12-0.2-xen Initrd image: /boot/initrd-2.6.31.12-0.2-xen-obs_worker
This assumes you have a VG dedicated to workers called "OBS"
Edit the file /etc/sysconfig/obs-worker in order to point to correct repository server.
nano /etc/sysconfig/obs-worker OBS_SRC_SERVER="cbe:5352" OBS_REPO_SERVERS="cbe:5252" OBS_VM_TYPE="xen" OBS_VM_KERNEL="/boot/vmlinuz-2.6.31.12-0.2-xen" OBS_VM_INITRD="/boot/initrd-2.6.31.12-0.2-xen-obs_worker" OBS_VM_DISK_AUTOSETUP_ROOT_FILESIZE="8192" OBS_VM_DISK_AUTOSETUP_SWAP_FILESIZE="2048" OBS_INSTANCE_MEMORY="1024" OBS_STORAGE_AUTOSETUP="yes" OBS_SETUP_WORKER_PARTITIONS="use_obs_vg" OBS_WORKER_ROOT_SIZE="8192" OBS_WORKER_SWAP_SIZE="2048"
The obsstoragesetup will wipe the OBS VG and create root/swap LVs for each worker
rcobsstoragesetup start
Output:
mdadm: No arrays found in config file or automatically
Waiting for udev to settle...
Scanning for LVM volume groups...
Reading all physical volumes. This may take a while...
Found volume group "OBS" using metadata type lvm2
Activating LVM volume groups...
0 logical volume(s) in volume group "OBS" now active
done
Logical volume "worker_root_1" created
Logical volume "worker_swap_1" created
Logical volume "worker_root_2" created
Logical volume "worker_swap_2" created
Logical volume "worker_root_3" created
Logical volume "worker_swap_3" created
Logical volume "worker_root_4" created
Logical volume "worker_swap_4" created
Logical volume "worker_root_5" created
Logical volume "worker_swap_5" created
Logical volume "worker_root_6" created
Logical volume "worker_swap_6" created
Logical volume "worker_root_7" created
Logical volume "worker_swap_7" created
Logical volume "worker_root_8" created
Logical volume "worker_swap_8" created
Logical volume "worker_root_9" created
Logical volume "worker_swap_9" created
Logical volume "worker_root_10" created
Logical volume "worker_swap_10" created
Logical volume "worker_root_11" created
Logical volume "worker_swap_11" created
Logical volume "worker_root_12" created
Logical volume "worker_swap_12" created
Logical volume "worker_root_13" created
Logical volume "worker_swap_13" created
Logical volume "worker_root_14" created
Logical volume "worker_swap_14" created
Logical volume "worker_root_15" created
Logical volume "worker_swap_15" created
Logical volume "worker_root_16" created
Logical volume "worker_swap_16" created
Logical volume "cache" created
mke2fs 1.41.9 (22-Aug-2009)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
3555328 inodes, 14201856 blocks
710092 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=4294967296
434 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208,
4096000, 7962624, 11239424
Writing inode tables: done
Creating journal (32768 blocks): done
Writing superblocks and filesystem accounting information: done
This filesystem will be automatically checked every 39 mounts or
180 days, whichever comes first. Use tune2fs -c or -i to override.
Looking for OBS Server LVM Volume
Setup local storage
Looking for OBS Worker Cache LVM Volume
Setting up OBS Workers according to LVM Volumes
Found XEN virtualization
done
Test?
xm create -c /var/run/obs/worker/8/build/xen.conf name=build:root8 memory=40 disk=phy:/dev/mapper/OBS-worker_root8,hda1,w disk=phy:/dev/mapper/OBS-worker_swap8,hda2,w extra="init=/.build/initscript_qemu_vm panic=1 console=ttyS0
Start the worker service:
chkconfig --add obsworker rcobsworker start