-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathmake_exps_bash.py
184 lines (155 loc) · 6.39 KB
/
make_exps_bash.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
import os
os.makedirs('experiments_FFNO/cluster3', exist_ok=True)
os.makedirs('experiments_FFNO/cluster4', exist_ok=True)
os.makedirs('experiments_FFNO/cluster5', exist_ok=True)
os.makedirs('experiments_FFNO/MAIL10', exist_ok=True)
os.makedirs('experiments_FFNO/MAIL15', exist_ok=True)
os.makedirs('experiments_FFNO/MAIL25', exist_ok=True)
bash_init = '#!/bin/bash\n'
train_command = 'python Train_FFNO.py'
data_dir = '/home/pouya/FPNO_data'
# templates ############################################################################################################
def make_cmd_1d_adv(beta, gpu_index):
cmd = f'CUDA_VISIBLE_DEVICES={gpu_index} {train_command}'
cmd += f' --name 1D_Advection_Beta{beta}'
cmd += f' --data_dir {data_dir}'
cmd += f' --data_train 1D_Advection_Beta{beta}.hdf5'
cmd += f' --data_ndims 1'
cmd += f' --model_configs ./configs_FFNO/pretrain.yaml'
cmd += '\n'
return cmd
def make_cmd_2d_adv(beta, gpu_index, seed):
cmd = f'CUDA_VISIBLE_DEVICES={gpu_index} {train_command}'
cmd += f' --name 2D_Advection_Beta{beta}_seed{seed}'
cmd += f' --data_dir {data_dir}'
cmd += f' --data_train 2D_Advection_Beta{beta}.hdf5'
cmd += f' --data_ndims 2'
cmd += f' --model_configs ./configs_FFNO/downstream_seed{seed}.yaml'
cmd += f' --transfer_from ./results/1D_Advection_Beta{beta}'
cmd += f' --transfer_configs ./configs_FFNO_transfer/u_1to2.yaml'
cmd += '\n'
return cmd
def make_cmd_1d_diff(nu, gpu_index):
cmd = f'CUDA_VISIBLE_DEVICES={gpu_index} {train_command}'
cmd += f' --name 1D_Diffusion_Nu{nu}'
cmd += f' --data_dir {data_dir}'
cmd += f' --data_train 1D_Diffusion_Nu{nu}.hdf5'
cmd += f' --data_ndims 1'
cmd += f' --model_configs ./configs_FFNO/pretrain.yaml'
cmd += '\n'
return cmd
def make_cmd_2d_diff(nu, gpu_index, seed):
cmd = f'CUDA_VISIBLE_DEVICES={gpu_index} {train_command}'
cmd += f' --name 2D_Diffusion_Nu{nu}_seed{seed}'
cmd += f' --data_dir {data_dir}'
cmd += f' --data_train 2D_Diffusion_Nu{nu}.hdf5'
cmd += f' --data_ndims 2'
cmd += f' --model_configs ./configs_FFNO/downstream_seed{seed}.yaml'
cmd += f' --transfer_from ./results/1D_Diffusion_Nu{nu}'
cmd += f' --transfer_configs ./configs_FFNO_transfer/u_1to2.yaml'
cmd += '\n'
return cmd
def make_automated_tmux(pc, n_gpus):
cmd = bash_init
for i in range(n_gpus):
cmd += f"tmux new-session -d -s gpu{i}\n"
cmd += f"tmux send-keys -t gpu{i} 'conda activate PDE_py3.9_torch1.10' C-m\n"
cmd += f"tmux send-keys -t gpu{i} 'bash experiments_FFNO/{pc}/run_gpu{i}.sh' C-m\n"
cmd += f"tmux ls\n"
return cmd
# cluster 4 ############################################################################################################
pc = 'cluster4' # Adv 0.1 and 0.4
# 1D_Adv_Beta0.1 and 1D_Adv_Beta0.4 on GPU 0:
for i, gpu_index in enumerate([0]):
cmd = bash_init
for beta in [0.1, 0.4]:
cmd += make_cmd_1d_adv(beta, gpu_index)
with open(f'experiments_FFNO/{pc}/run_1D_Adv_{0.1}_{0.4}.sh', 'w') as file:
file.write(cmd)
# 2D_Adv_Beta0.1 on GPUs 0, 1, 2:
for i, gpu_index in enumerate([0, 1, 2]):
beta = 0.1
cmd = bash_init
cmd += make_cmd_2d_adv(beta, gpu_index, i)
with open(f'experiments_FFNO/{pc}/run_gpu{gpu_index}.sh', 'w') as file:
file.write(cmd)
# 2D_Adv_Beta0.4 on GPUs 3, 4, 5:
for i, gpu_index in enumerate([3, 4, 5]):
beta = 0.4
cmd = bash_init
cmd += make_cmd_2d_adv(beta, gpu_index, i)
with open(f'experiments_FFNO/{pc}/run_gpu{gpu_index}.sh', 'w') as file:
file.write(cmd)
# Tmux automation
cmd = make_automated_tmux(pc, 6)
with open(f'experiments_FFNO/{pc}/run_tmux.sh', 'w') as file:
file.write(cmd)
# cluster 5 ###########################################################################################################
pc = 'cluster5' # Adv 1.0 and Diff 0.004
# 1D_Adv_Beta0.4 and 1D_Diff_Nu0.004 on GPU 0:
for i, gpu_index in enumerate([0]):
cmd = bash_init
for beta in [1.0]:
cmd += make_cmd_1d_adv(beta, gpu_index)
for nu in [0.004]:
cmd += make_cmd_1d_diff(nu, gpu_index)
with open(f'experiments_FFNO/{pc}/run_1D_Adv_{beta}_Diff_{nu}.sh', 'w') as file:
file.write(cmd)
# 2D_Adv_Beta1.0 on GPUs 0, 1, 2:
for i, gpu_index in enumerate([0, 1, 2]):
beta = 1.0
cmd = bash_init
cmd += make_cmd_2d_adv(beta, gpu_index, i)
with open(f'experiments_FFNO/{pc}/run_gpu{gpu_index}.sh', 'w') as file:
file.write(cmd)
# 2D_Diff_Nu0.004 on GPUs 3, 4, 5:
for i, gpu_index in enumerate([3, 4, 5]):
nu = 0.004
cmd = bash_init
cmd += make_cmd_2d_diff(0.004, gpu_index, i)
with open(f'experiments_FFNO/{pc}/run_gpu{gpu_index}.sh', 'w') as file:
file.write(cmd)
# Tmux automation
cmd = make_automated_tmux(pc, 6)
with open(f'experiments_FFNO/{pc}/run_tmux.sh', 'w') as file:
file.write(cmd)
# MAIL-10 ##############################################################################################################
pc = 'MAIL10' # Diff 0.001
# 1D_Diff_Nu0.001 on GPU 0:
for i, gpu_index in enumerate([0]):
cmd = bash_init
for nu in [0.001]:
cmd += make_cmd_1d_diff(nu, gpu_index)
with open(f'experiments_FFNO/{pc}/run_1D_Diff_{nu}.sh', 'w') as file:
file.write(cmd)
# 2D_Diff_Nu0.001 on GPUs 0, 1, 2:
for i, gpu_index in enumerate([0, 1, 2]):
nu = 0.001
cmd = bash_init
cmd += make_cmd_2d_diff(0.001, gpu_index, i)
with open(f'experiments_FFNO/{pc}/run_gpu{gpu_index}.sh', 'w') as file:
file.write(cmd)
# Tmux automation
cmd = make_automated_tmux(pc, 3)
with open(f'experiments_FFNO/{pc}/run_tmux.sh', 'w') as file:
file.write(cmd)
# MAIL-15 ##############################################################################################################
pc = 'MAIL15' # Diff 0.002
# 1D_Diff_Nu0.002 on GPU 0:
for i, gpu_index in enumerate([0]):
cmd = bash_init
for nu in [0.002]:
cmd += make_cmd_1d_diff(nu, gpu_index)
with open(f'experiments_FFNO/{pc}/run_1D_Diff_Nu_{0.002}.sh', 'w') as file:
file.write(cmd)
# 2D_Diff_Nu0.002 on GPUs 0, 1, 2:
for i, gpu_index in enumerate([0, 1, 2]):
nu = 0.002
cmd = bash_init
cmd += make_cmd_2d_diff(nu, gpu_index, i)
with open(f'experiments_FFNO/{pc}/run_gpu{gpu_index}.sh', 'w') as file:
file.write(cmd)
# Tmux automation
cmd = make_automated_tmux(pc, 3)
with open(f'experiments_FFNO/{pc}/run_tmux.sh', 'w') as file:
file.write(cmd)