Annotation of /virt-cluster/vc/ogs.py

Parent Directory | Revision Log

---

Revision 1 - (view) (download) (as text)

1 :	agomez	1	import os
2 :			import subprocess
3 :			import tempfile
4 :			import vcutil
5 :			import xml.dom.minidom
6 :
7 :			def add_administration_host(hostname):
8 :			vcutil.execute("qconf -ah %s" % hostname)
9 :
10 :			def delete_administration_host(hostname):
11 :			vcutil.execute("qconf -dh %s" % hostname)
12 :
13 :			def get_administration_hosts():
14 :			hosts = []
15 :			try:
16 :			output,err = vcutil.execute("qconf -sh")
17 :			hosts = [host.strip() for host in output.split('\n') if len(host.strip())>0]
18 :			except vcutil.CommandError as e:
19 :			if not "no submit host defined" in e.output:
20 :			raise e
21 :			return hosts
22 :
23 :			def add_execution_host(hostname):
24 :			template = """hostname %s
25 :			load_scaling NONE
26 :			complex_values NONE
27 :			user_lists NONE
28 :			xuser_lists NONE
29 :			projects NONE
30 :			xprojects NONE
31 :			usage_scaling NONE
32 :			report_variables NONE
33 :			""" % hostname
34 :			file = open(tempfile.mkstemp()[1],'w')
35 :			file.write(template)
36 :			file.close()
37 :			vcutil.execute("qconf -Ae %s" % file.name)
38 :			os.remove(file.name)
39 :
40 :			def delete_execution_host(hostname):
41 :			vcutil.execute("qconf -de %s" % hostname)
42 :
43 :			def get_execution_hosts():
44 :			hosts = []
45 :			try:
46 :			output,err = vcutil.execute("qconf -sel")
47 :			hosts = [host.strip() for host in output.split('\n') if len(host.strip())>0]
48 :			except vcutil.CommandError as e:
49 :			if not "no execution host defined" in e.output:
50 :			raise e
51 :			return hosts
52 :
53 :			def add_submit_host(hostname):
54 :			vcutil.execute("qconf -as %s" % hostname)
55 :
56 :			def delete_submit_host(hostname):
57 :			vcutil.execute("qconf -ds %s" % hostname)
58 :
59 :			def get_submit_hosts():
60 :			hosts = []
61 :			try:
62 :			output,err = vcutil.execute("qconf -ss")
63 :			hosts = [host.strip() for host in output.split('\n') if len(host.strip())>0]
64 :			except vcutil.CommandError as e:
65 :			if not "no submit host defined" in e.output:
66 :			raise e
67 :			return hosts
68 :
69 :			def get_group(groupname):
70 :			hosts = []
71 :			output,err = vcutil.execute("qconf -shgrp %s" % groupname)
72 :			hosts = [host for host in output.split('\n',1)[1].split()[1:] if len(host.strip('\\'))>0]
73 :			if "NONE" in hosts:
74 :			hosts.remove("NONE")
75 :			return hosts
76 :
77 :			def add_to_group(groupname,hostname):
78 :			hosts = get_group(groupname)
79 :			if not hostname in hosts:
80 :			hosts.append(hostname)
81 :			template = """group_name %s
82 :			hostlist %s""" % (groupname," ".join(hosts))
83 :			file = open(tempfile.mkstemp()[1],'w')
84 :			file.write(template)
85 :			file.close()
86 :			vcutil.execute("qconf -Mhgrp %s" % file.name)
87 :			os.remove(file.name)
88 :
89 :			def delete_from_group(groupname,hostname):
90 :			hosts = get_group(groupname)
91 :			if hostname in hosts:
92 :			hosts.remove(hostname)
93 :			if len(hosts) == 0:
94 :			hosts.append("NONE")
95 :			template = """group_name %s
96 :			hostlist %s""" % (groupname," ".join(hosts))
97 :			file = open(tempfile.mkstemp()[1],'w')
98 :			file.write(template)
99 :			file.close()
100 :			vcutil.execute("qconf -Mhgrp %s" % file.name)
101 :			os.remove(file.name)
102 :
103 :			"""
104 :			def get_jobs_at_host(hostname):
105 :			output,err = vcutil.execute("qstat -xml -q *@%s" % hostname)
106 :			dom = xml.dom.minidom.parseString(output)
107 :			jobs = [node.firstChild.nodeValue for node in dom.getElementsByTagName("JB_job_number") ]
108 :			return jobs
109 :			"""
110 :
111 :			def get_tasks_at_host(hostname):
112 :			tasks = []
113 :			output,err = vcutil.execute("qstat -xml -f -g d -q *@%s" % hostname, ignore_error = True)
114 :			try:
115 :			dom = xml.dom.minidom.parseString(output)
116 :			except:
117 :			return tasks
118 :			for instance in [ ql for ql in dom.getElementsByTagName("Queue-List") if hostname in ql.getElementsByTagName("name")[0].firstChild.nodeValue ]:
119 :			for job in instance.getElementsByTagName("job_list"):
120 :			jobid = job.getElementsByTagName("JB_job_number")[0].firstChild.nodeValue
121 :			taskid = None
122 :			if len(job.getElementsByTagName("tasks"))>0:
123 :			taskid = job.getElementsByTagName("tasks")[0].firstChild.nodeValue
124 :			tasks.append((jobid,taskid))
125 :			return tasks
126 :
127 :			def delete_job(jobid,taskid = None, force=False):
128 :			opts=""
129 :			if force:
130 :			opts = "-f"
131 :			if taskid:
132 :			jobid = "%s.%s" % (jobid,taskid)
133 :			vcutil.execute("qdel %s %s" % (opts,jobid,))
134 :
135 :			def resched_job(jobid,taskid = None, force=False):
136 :			opts=""
137 :			if force:
138 :			opts = "-f"
139 :			if taskid:
140 :			jobid = "%s.%s" % (jobid,taskid)
141 :			vcutil.execute("qmod %s -rj %s" % (opts,jobid,))
142 :
143 :			def get_queues():
144 :			queues = []
145 :			try:
146 :			output,err = vcutil.execute("qconf -sql")
147 :			queues = [queue.strip() for queue in output.split('\n') if len(queue.strip())>0]
148 :			except vcutil.CommandError as e:
149 :			if not "no cqueue list defined" in e.output:
150 :			raise e
151 :			return queues
152 :
153 :			def get_queue(queue_name):
154 :			output,err = vcutil.execute("qconf -sq %s" % queue_name)
155 :			queue = {}
156 :			output = "".join(output.split("\\\n"))
157 :			for line in output.split('\n'):
158 :			line = line.strip()
159 :			if len(line)>0:
160 :			key,value = line.split(None,1)
161 :			queue[key] = value
162 :			return queue
163 :
164 :			def modify_queue(queue_name,mods):
165 :			queue = get_queue(queue_name)
166 :			for key,value in mods.items():
167 :			queue[key] = value
168 :			file = open(tempfile.mkstemp()[1],'w')
169 :			file.write("\n".join([ "%s %s" % items for items in queue.items()]))
170 :			file.close()
171 :			vcutil.execute("qconf -Mq %s" % file.name)
172 :			os.remove(file.name)
173 :
174 :			def get_global():
175 :			output,err = vcutil.execute("qconf -sconf")
176 :			conf = {}
177 :			output = "".join(output.split("\\\n"))
178 :			for line in output.split('\n'):
179 :			line = line.strip()
180 :			if len(line)>0 and not line.startswith('#'):
181 :			key,value = line.split(None,1)
182 :			conf[key] = value
183 :			return conf
184 :
185 :			def modify_global(mods):
186 :			conf = get_global()
187 :			for key,value in mods.items():
188 :			conf[key] = value
189 :			file = open(tempfile.gettempdir()+'/global','w')
190 :			file.write("\n".join([ "%s %s" % items for items in conf.items()]))
191 :			file.write("\n")
192 :			file.close()
193 :			vcutil.execute("qconf -Mconf %s" % file.name)
194 :			os.remove(file.name)
195 :
196 :			def set_slots_host(queue_name,host,number):
197 :			slots = get_queue(queue_name)['slots']
198 :			slots = [s.strip().strip('[]') for s in slots.split(",") ]
199 :			if number<1:
200 :			number = 1
201 :			else:
202 :			number = int(number)
203 :			new_slots = []
204 :			added = False
205 :			for slot in slots:
206 :			if slot.split('=')[0] == host:
207 :			slot="%s=%i" % (host,number)
208 :			added = True
209 :			if "=" in slot:
210 :			slot = "[%s]" % (slot,)
211 :			new_slots.append(slot)
212 :			if not added:
213 :			new_slots.append("[%s=%i]"%(host,number))
214 :			modify_queue(queue_name,{'slots':",".join(new_slots)})
215 :
216 :			def del_slots_host(queue_name,host):
217 :			slots = get_queue(queue_name)['slots']
218 :			slots = [s.strip('[]') for s in slots.split(",") ]
219 :			new_slots = []
220 :			for slot in slots:
221 :			if slot.split('=')[0] == host:
222 :			continue
223 :			if "=" in slot:
224 :			slot = "[%s]" % (slot,)
225 :			new_slots.append("%s"%(slot,))
226 :			modify_queue(queue_name,{'slots':",".join(new_slots)})
227 :
228 :			def start_execd():
229 :			vcutil.execute("/etc/init.d/sgeexecd")
230 :
231 :			def start_sgemaster():
232 :			vcutil.execute("/etc/init.d/sgemaster")
233 :
234 :			def start_qmaster():
235 :			vcutil.execute("/etc/init.d/sgemaster -qmaster")
236 :
237 :			def start_shadowd():
238 :			vcutil.execute("/etc/init.d/sgemaster -shadowd")
239 :
240 :			def remove_host(hostname):
241 :			if hostname in get_execution_hosts():
242 :			delete_from_group("@allhosts",hostname)
243 :			for jobid,taskid in get_tasks_at_host(hostname):
244 :			# delete_job(jobid,taskid,force=True)
245 :			resched_job(jobid,taskid,force=True)
246 :			delete_execution_host(hostname)
247 :			delete_administration_host(hostname)
248 :			# del_slots_host('all.q',hostname)
249 :
250 :			def new_host(hostname, slots=1):
251 :			if not hostname in get_execution_hosts():
252 :			add_to_group("@allhosts",hostname)
253 :			add_administration_host(hostname)
254 :			set_slots_host('all.q',hostname,slots)
255 :
256 :			if __name__== "__main__":
257 :			print get_administration_hosts()
258 :			print "------"
259 :			print get_submit_hosts()
260 :			print "------"
261 :			print get_execution_hosts()
262 :			print "------"
263 :			group="@allhosts"
264 :			node="node253"
265 :			print get_group(group)
266 :			print "------"
267 :
268 :
269 :
270 :
271 :
272 :

---

root@forge.cesga.es	ViewVC Help
Powered by ViewVC 1.0.0