1 : |
agomez |
1 |
#!/usr/bin/python
|
2 : |
agomez |
9 |
#
|
3 : |
|
|
# BonFIRE Virtual Clusters on Federated Clouds Demonstration Kit
|
4 : |
|
|
#
|
5 : |
|
|
# Copyright (c) Fundacion Centro Tecnologico de Supercomputacion de Galicia 2012
|
6 : |
|
|
#
|
7 : |
|
|
# License GPL Version 3
|
8 : |
|
|
#
|
9 : |
|
|
# The research leading to these results has received funding from
|
10 : |
|
|
# the European Community's Seventh Frameqork Programme (FP7/2007-2013)
|
11 : |
|
|
# under agreement number 257386
|
12 : |
|
|
#
|
13 : |
|
|
# This software is provided with ABSOLUTELY NO WARRANTY
|
14 : |
|
|
#
|
15 : |
agomez |
1 |
import os
|
16 : |
|
|
import os.path
|
17 : |
|
|
import subprocess
|
18 : |
|
|
import sys
|
19 : |
|
|
import time
|
20 : |
|
|
|
21 : |
|
|
from logger import log,configure
|
22 : |
|
|
if len(sys.argv)>1:
|
23 : |
|
|
configure(logfile = sys.argv[1], debug=True, console=False)
|
24 : |
|
|
else:
|
25 : |
|
|
configure(debug=True, console=True)
|
26 : |
|
|
|
27 : |
|
|
try:
|
28 : |
|
|
import bonfire
|
29 : |
|
|
import hostsfile
|
30 : |
|
|
import ogs
|
31 : |
|
|
import vcutil
|
32 : |
|
|
|
33 : |
|
|
allow_single_cluster = True
|
34 : |
|
|
default_bonfire = "/etc/default/bonfire"
|
35 : |
|
|
bonfire.read_defaults(default_bonfire)
|
36 : |
|
|
hosts_file = "/etc/hosts"
|
37 : |
|
|
log.info("Bonfire URI: %s" % bonfire.uri)
|
38 : |
|
|
log.info("Credentials: %s:%s" % (bonfire.user,bonfire.password,))
|
39 : |
|
|
log.info("Experiment: %s" % bonfire.experiment_id)
|
40 : |
|
|
|
41 : |
|
|
#Basic hostsfile for main nodes
|
42 : |
|
|
while True:
|
43 : |
|
|
try:
|
44 : |
|
|
log.info("Get experiment")
|
45 : |
|
|
master_host = None
|
46 : |
|
|
shadow_host = None
|
47 : |
|
|
while (master_host == None) and ((not allow_single_cluster) or (shadow_host == None)) :
|
48 : |
|
|
experiment = bonfire.get_experiment(bonfire.experiment_id)
|
49 : |
|
|
for compute in experiment['computes']:
|
50 : |
|
|
if compute["hostname"].startswith("master"):
|
51 : |
|
|
master_host = (compute['ip'],compute['hostname'])
|
52 : |
|
|
elif compute["hostname"].startswith("shadow"):
|
53 : |
|
|
shadow_host = (compute['ip'],compute['hostname'])
|
54 : |
|
|
main_hosts = [('127.0.0.1','localhost'),(bonfire.wan_ip,bonfire.hostname), master_host]
|
55 : |
|
|
if shadow_host:
|
56 : |
|
|
main_hosts.append(shadow_host)
|
57 : |
|
|
data_host = master_host
|
58 : |
|
|
if bonfire.hostname.startswith("client-shadow"):
|
59 : |
|
|
data_host = shadow_host
|
60 : |
|
|
log.debug(str(main_hosts))
|
61 : |
|
|
break
|
62 : |
|
|
except Exception as excpt:
|
63 : |
|
|
log.exception(excpt)
|
64 : |
|
|
time.sleep(20)
|
65 : |
|
|
|
66 : |
|
|
hostsfile.store(hosts_file,main_hosts)
|
67 : |
|
|
|
68 : |
|
|
#Try to mount shared
|
69 : |
|
|
log.info("Try to mount NFS volume")
|
70 : |
|
|
while True:
|
71 : |
|
|
try:
|
72 : |
|
|
log.debug(vcutil.execute("mount -t nfs %s:/volume /shared" % data_host[1]))
|
73 : |
|
|
break
|
74 : |
|
|
except Exception as excpt:
|
75 : |
|
|
log.debug("NFS volume not available")
|
76 : |
|
|
log.exception(excpt)
|
77 : |
|
|
try:
|
78 : |
|
|
log.debug(vcutil.execute("umount /shared"))
|
79 : |
|
|
except Exception as excpt:
|
80 : |
|
|
log.exception(excpt)
|
81 : |
|
|
time.sleep(20)
|
82 : |
|
|
|
83 : |
|
|
#Mount shared
|
84 : |
|
|
log.debug(vcutil.execute("umount /shared"))
|
85 : |
|
|
log.info("Sync mount NFS volume")
|
86 : |
|
|
log.debug(vcutil.execute("mount -t nfs -o nordirplus,hard,nointr,rw %s:/volume /shared" % data_host[1]))
|
87 : |
|
|
|
88 : |
|
|
while not (os.path.exists("/shared/ogs/default/common/sgeexecd") and os.path.exists("/shared/ogs/default/common/settings.sh")):
|
89 : |
|
|
log.debug("OGS directory not available")
|
90 : |
|
|
time.sleep(5)
|
91 : |
|
|
|
92 : |
|
|
#Configure OGS installation
|
93 : |
|
|
log.debug(vcutil.execute("""export SGE_ROOT=/shared/ogs
|
94 : |
|
|
cd $SGE_ROOT
|
95 : |
|
|
echo \". $SGE_ROOT/default/common/settings.sh\" >> /root/.bashrc
|
96 : |
|
|
ln -s $SGE_ROOT/default/common/sgeexecd /etc/init.d/sgeexecd
|
97 : |
|
|
"""))
|
98 : |
|
|
|
99 : |
|
|
log.debug("Checking if OGS is available")
|
100 : |
|
|
while True:
|
101 : |
|
|
try:
|
102 : |
|
|
log.debug(vcutil.execute(". /shared/ogs/default/common/settings.sh; qstat"))
|
103 : |
|
|
break
|
104 : |
|
|
except Exception as excpt:
|
105 : |
|
|
log.exception(excpt)
|
106 : |
|
|
time.sleep(10)
|
107 : |
|
|
|
108 : |
|
|
log.debug(vcutil.execute(". /shared/ogs/default/common/settings.sh; python -u /root/vc/vc-node-execd.py /var/log/vc-node-execd.log" , fork = True))
|
109 : |
|
|
except Exception as excpt:
|
110 : |
|
|
log.exception(excpt)
|
111 : |
|
|
|