Skip to content

Commit b457e04

Browse files
author
spigad
committed
first commit of all developed templates. CMS-HTCondor* are meant to show several corner cases
1 parent d2bc2ac commit b457e04

9 files changed

+1135
-37
lines changed

‎dodas/AMS_cluster.yaml

+362
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,362 @@
1+
tosca_definitions_version: tosca_simple_yaml_1_0
2+
3+
imports:
4+
- indigo_custom_types: https://raw.githubusercontent.com/indigo-dc/tosca-types/dodas_roll/custom_types.yaml
5+
6+
7+
description: TOSCA example for specifying a Mesos Cluster with a containerized HTCondor batch system
8+
9+
topology_template:
10+
11+
inputs:
12+
13+
marathon_username:
14+
type: string
15+
default: "admin"
16+
17+
marathon_password:
18+
type: string
19+
default: "passwd"
20+
21+
number_of_slaves:
22+
type: integer
23+
default: 1
24+
25+
number_of_masters:
26+
type: integer
27+
default: 1
28+
29+
number_of_wn_instances:
30+
type: integer
31+
default: 1
32+
33+
cpu_x_wn:
34+
type: float
35+
default: 1.0
36+
37+
ram_x_wn:
38+
type: float
39+
default: 4000.0
40+
41+
docker_cpu_x_wn:
42+
type: float
43+
default: 0.95
44+
45+
docker_ram_x_wn:
46+
type: float
47+
default: 3998.95
48+
49+
# Image ID. IaaS depending.
50+
server_image:
51+
type: string
52+
# default: "ost://cloud.recas.ba.infn.it/1113d7e8-fc5d-43b9-8d26-61906d89d479"
53+
default: "Ubuntu_16.04"
54+
55+
# IAM condfiguration to enable GSI HTCondor agent authentication
56+
iam_token:
57+
type: string
58+
default: "IAM-valid-Token-Here"
59+
60+
iam_client_id:
61+
type: string
62+
default: "IAM Client ID"
63+
64+
iam_client_secret:
65+
type: string
66+
default: "IAM-Client-Secret"
67+
68+
proxycache_image:
69+
type: string
70+
default: "dodasts/ttscache"
71+
72+
# Parameters repo
73+
ams_repo_server_url:
74+
type: string
75+
default: "http://IP-HERE/cvmfs/ams.local.repo"
76+
77+
ams_repo_public_key_path:
78+
type: string
79+
default: "/etc/cvmfs/keys/ams.local.repo.pub"
80+
81+
ams_repo_repository_name:
82+
type: string
83+
default: "ams.local.repo"
84+
85+
ams_repo_public_key_url:
86+
type: string
87+
default: "https://gist.githubusercontent.com/spigad/b19fa3a3e32bcc63ce6bc1513a9bc4aa/raw/0dd06039513fa194775adb87ab60891fb55903d4/ams-cvmfs-google"
88+
89+
ams_repo_public_key:
90+
type: string
91+
default: |
92+
-----BEGIN PUBLIC KEY-----
93+
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4ZnSyYfHJSZ4qiKsVW99
94+
/xcrEZgv2auLLRM4JClsy87nINClaujAONAOK64308ZhJXVMx9z0PywZ6Ngy2TO1
95+
rjrissJP0LBavxm2SuPKWep9lQczVkOtQsWNN1YoKDF6jMvHCtIOROZuHZniDr0U
96+
CZCq2j7zni2t0kbBErpry0M+wTtkOIMcTPI6joem8JcRQ8NU915eWCAiwbe5C4Gj
97+
dOSZdlvqN+zEUWc+ZlgMqNnqyda1QD0msoiCT+DfKn2B0O+EqVjw1IHyHOVEc9QS
98+
X9rqkIIcnYfK8gPE2eA6BOFF0TMzW7zrcLd2B8LkSIhurhPFRJRz8ZtekOOr21J8
99+
jwIDAQAB
100+
-----END PUBLIC KEY-----
101+
102+
node_templates:
103+
104+
mesos_master:
105+
type: tosca.nodes.indigo.MesosMaster
106+
properties:
107+
marathon_username: { get_input: marathon_username }
108+
marathon_password: { get_input: marathon_password }
109+
chronos_password: test_pass
110+
mesos_masters_list: { get_attribute: [ mesos-master-server, private_address ] }
111+
requirements:
112+
- host: mesos-master-server
113+
114+
mesos_slave:
115+
type: tosca.nodes.indigo.MesosSlave
116+
properties:
117+
master_ips: { get_attribute: [ mesos-master-server, private_address ] }
118+
front_end_ip: { get_attribute: [ mesos-master-server, private_address, 0 ] }
119+
requirements:
120+
- host: mesos-slave-server
121+
122+
mesos_slaveschedd:
123+
type: tosca.nodes.indigo.MesosSlave
124+
properties:
125+
master_ips: { get_attribute: [ mesos-master-server, private_address ] }
126+
front_end_ip: { get_attribute: [ mesos-master-server, private_address, 0 ] }
127+
requirements:
128+
- host: mesos-slaveschedd-server
129+
130+
mesos_slaveccb:
131+
type: tosca.nodes.indigo.MesosSlave
132+
properties:
133+
master_ips: { get_attribute: [ mesos-master-server, private_address ] }
134+
front_end_ip: { get_attribute: [ mesos-master-server, private_address, 0 ] }
135+
requirements:
136+
- host: mesos-ccb-server
137+
138+
mesos_load_balancer:
139+
type: tosca.nodes.indigo.MesosLoadBalancer
140+
properties:
141+
marathon_username: { get_input: marathon_username }
142+
master_ips: { get_attribute: [ mesos-master-server, private_address ] }
143+
marathon_password: { get_input: marathon_password }
144+
requirements:
145+
- host: mesos-lb-server
146+
- dependency: mesos_master
147+
148+
ams_services:
149+
type: tosca.nodes.indigo.AmsCondorMasterConfig
150+
properties:
151+
marathon_username: { get_input: marathon_username }
152+
marathon_password: { get_input: marathon_password }
153+
master_ips: { get_attribute: [ mesos-master-server, private_address ] }
154+
ams_default_squid: { get_attribute: [ mesos-lb-server, private_address, 0 ] }
155+
requirements:
156+
- host: mesos_master
157+
158+
ams_executor:
159+
type: tosca.nodes.indigo.AmsCondorExecuteConfig
160+
properties:
161+
marathon_username: { get_input: marathon_username }
162+
marathon_password: { get_input: marathon_password }
163+
master_ips: { get_attribute: [ mesos-master-server, private_address ] }
164+
htcondor_config_schedd_ip: { get_attribute: [ mesos-slaveschedd-server, private_address, 0 ] }
165+
htcondor_config_ccb_ip: { get_attribute: [ mesos-ccb-server, private_address, 0 ] }
166+
proxycache_host: { get_attribute: [ mesos-lb-server, private_address, 0 ] }
167+
htcondor_config_condor_host: { get_attribute: [ mesos-ccb-server, private_address, 0 ]}
168+
htcondor_config_ccb_address: { get_attribute: [ mesos-ccb-server, public_address, 0 ]}
169+
htcondor_config_condor_image: "dodasts/ams"
170+
requirements:
171+
- host: mesos_master
172+
- dependency: htcondor_services
173+
174+
htcondor_services:
175+
type: tosca.nodes.indigo.HTCondorServices
176+
properties:
177+
marathon_username: { get_input: marathon_username }
178+
marathon_password: { get_input: marathon_password }
179+
htcondor_config_condor_host: { get_attribute: [ mesos-ccb-server, private_address, 0 ]}
180+
htcondor_config_ccb_address: { get_attribute: [ mesos-ccb-server, public_address, 0 ]}
181+
htcondor_config_high_port: "32048"
182+
htcondor_config_low_port: "31024"
183+
master_ips: { get_attribute: [ mesos-master-server, private_address ] }
184+
lb_ips: { get_attribute: [ mesos-lb-server, private_address ] }
185+
htcondor_config_schedd_ip_pub: { get_attribute: [ mesos-slaveschedd-server, public_address, 0 ] }
186+
htcondor_config_schedd_ip: { get_attribute: [ mesos-slaveschedd-server, private_address, 0 ] }
187+
htcondor_config_ccb_ip: { get_attribute: [ mesos-ccb-server, private_address, 0 ] }
188+
config_mode: "master"
189+
htcondor_config_condor_image: "dodasts/ams"
190+
proxycache_host: { get_attribute: [ mesos-lb-server, private_address, 0 ] }
191+
iam_access_token: { get_input: iam_token }
192+
iam_client_id: { get_input: iam_client_id }
193+
iam_client_secret: { get_input: iam_client_secret }
194+
proxycache_image: { get_input: proxycache_image }
195+
requirements:
196+
- host: mesos_master
197+
198+
condor_wn:
199+
type: tosca.nodes.indigo.AmsCondorWnConfig
200+
properties:
201+
marathon_username: { get_input: marathon_username }
202+
marathon_password: { get_input: marathon_password }
203+
master_ips: { get_attribute: [ mesos-master-server, private_address ] }
204+
number_of_wn_instances: { get_input: number_of_wn_instances }
205+
number_of_slaves: { get_input: number_of_slaves }
206+
cpu_x_wn: { get_input: cpu_x_wn }
207+
ram_x_wn: { get_input: ram_x_wn }
208+
docker_cpu_x_wn: { get_input: docker_cpu_x_wn }
209+
docker_ram_x_wn: { get_input: docker_ram_x_wn }
210+
ams_default_squid: { get_attribute: [ mesos-lb-server, private_address, 0 ] }
211+
ams_repo_server_url: { get_input: ams_repo_server_url }
212+
ams_repo_public_key_path: { get_input: ams_repo_public_key_path }
213+
ams_repo_repository_name: { get_input: ams_repo_repository_name }
214+
ams_repo_public_key_url: { get_input: ams_repo_public_key_url }
215+
ams_repo_public_key: { get_input: ams_repo_public_key }
216+
requirements:
217+
- host: mesos_slave
218+
219+
##
220+
# Resource section
221+
# Numer of instances and image parameters
222+
mesos-master-server:
223+
type: tosca.nodes.indigo.Compute
224+
capabilities:
225+
endpoint:
226+
properties:
227+
network_name: PUBLIC
228+
dns_name: mesosserverpublic
229+
ports:
230+
mesos_port:
231+
protocol: tcp
232+
source: 5050
233+
marathon_port:
234+
protocol: tcp
235+
source: 8443
236+
scalable:
237+
properties:
238+
count: 1
239+
host:
240+
properties:
241+
num_cpus: 4
242+
mem_size: 8 GB
243+
os:
244+
properties:
245+
image: { get_input: server_image }
246+
247+
mesos-slave-server:
248+
type: tosca.nodes.indigo.Compute
249+
capabilities:
250+
scalable:
251+
properties:
252+
count: 1
253+
host:
254+
properties:
255+
num_cpus: 2
256+
mem_size: 4 GB
257+
os:
258+
properties:
259+
image: { get_input: server_image }
260+
261+
mesos-slaveschedd-server:
262+
type: tosca.nodes.indigo.Compute
263+
capabilities:
264+
endpoint:
265+
properties:
266+
network_name: PUBLIC
267+
dns_name: scheddpublic
268+
ports:
269+
schedd_port:
270+
protocol: tcp
271+
source: 9618
272+
webui_port:
273+
protocol: tcp
274+
source: 48080
275+
shared_ports:
276+
protocol: tcp
277+
source_range: [ 31024, 32048 ]
278+
scalable:
279+
properties:
280+
count: 1
281+
host:
282+
properties:
283+
num_cpus: 4
284+
mem_size: 8 GB
285+
os:
286+
properties:
287+
image: { get_input: server_image }
288+
requirements:
289+
# contextually this can only be a relationship type
290+
- local_storage:
291+
# capability is provided by Compute Node Type
292+
node: condor_spool_dir
293+
capability: tosca.capabilities.Attachment
294+
relationship:
295+
type: tosca.relationships.AttachesTo
296+
properties:
297+
location: /spooldir
298+
device: hdb
299+
300+
condor_spool_dir:
301+
type: tosca.nodes.indigo.BlockStorage
302+
properties:
303+
size: 500 GB
304+
type: pd-ssd # syntax here is IaaS specific. This is a working exaple for Google Cloud.
305+
306+
# Aleternatively this use just std disk (no QoS selection)
307+
# condor_spool_dir:
308+
# type: tosca.nodes.BlockStorage
309+
# properties:
310+
# size: 500 GB
311+
312+
mesos-ccb-server:
313+
type: tosca.nodes.indigo.Compute
314+
capabilities:
315+
endpoint:
316+
properties:
317+
network_name: PUBLIC
318+
dns_name: ccb
319+
ports:
320+
schedd_port:
321+
protocol: tcp
322+
source: 9618
323+
scalable:
324+
properties:
325+
count: 1
326+
host:
327+
properties:
328+
num_cpus: 8
329+
mem_size: 16 GB
330+
os:
331+
properties:
332+
image: { get_input: server_image }
333+
334+
mesos-lb-server:
335+
type: tosca.nodes.indigo.Compute
336+
capabilities:
337+
endpoint:
338+
properties:
339+
network_name: PUBLIC
340+
dns_name: mesoslb
341+
scalable:
342+
properties:
343+
count: 1
344+
host:
345+
properties:
346+
num_cpus: 2
347+
mem_size: 4 GB
348+
os:
349+
properties:
350+
image: { get_input: server_image }
351+
352+
outputs:
353+
mesos_lb_ip:
354+
value: { get_attribute: [ mesos-lb-server, private_address ] }
355+
mesos_endpoint:
356+
value: { concat: [ 'http://', get_attribute: [ mesos-master-server, public_address, 0 ], ':5050' ] }
357+
marathon_endpoint:
358+
value: { concat: [ 'https://', get_attribute: [ mesos-master-server, public_address, 0 ], ':8443' ] }
359+
ccb_endpoint:
360+
value: { concat: [ 'condor_status ', get_attribute: [ mesos-ccb-server, public_address, 0 ], ':9618 -schedd' ] }
361+
register_endpoint:
362+
value: { concat: [ 'http://', get_attribute: [ mesos-slaveschedd-server, public_address, 0 ], ':48080/register' ] }

‎dodas/Basic.yaml

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
tosca_definitions_version: tosca_simple_yaml_1_0
22

33
imports:
4-
# - indigo_custom_types: https://raw.githubusercontent.com/indigo-dc/tosca-types/master/custom_types.yaml
5-
- indigo_custom_types: https://gist.githubusercontent.com/dciangot/f3b20516252659b27dfb46d0d313fa2e/raw/9f111ed0a3c27f3397dcd20dc704e56b6bb4641b/tosca.yaml
4+
- indigo_custom_types: https://raw.githubusercontent.com/indigo-dc/tosca-types/dodas_roll/custom_types.yaml
65

76

87
description: >

0 commit comments

Comments
 (0)