suite.rc 11.7 KB
Newer Older
1 2 3 4 5 6
#!jinja2
[cylc]
	UTC mode = True # Ignore DST
	abort if any task fails = False

[scheduling]
7
	initial cycle point = {{INITIAL_START_DATE}}
8
{% if FINAL_CYCLE_POINT != 'NONE' %}
9
        final cycle point = {{FINAL_CYCLE_POINT}}
10
{% endif %}
11 12 13 14 15 16
	[[special tasks]]
		clock-trigger = start_cycle(PT0M)
	[[dependencies]]
		[[[R1]]]
			graph = """
					copy_common_to_remote => softlink_forcing_remote
17
{% if COLD_START and BOUNDARY_FORCING == 'CMEMS' %}
18 19 20
					write_run_namelist => adjust_namelist & generate_CMEMS_start => mv_start_file => run_fvcom
{% elif COLD_START and BOUNDARY_FORCING == 'NEST_FILE' %}
					write_run_namelist => adjust_namelist & generate_nest_start => mv_start_file => run_fvcom
21 22
{% elif ARCHIVE_RESTART %}
					transfer_archive_restart => run_fvcom
23 24
{% else %}
					transfer_restart => run_fvcom
25
{% endif %}
26
					"""
27
		[[[P{{CYCLE_FREQ}}D]]]
28
			graph = """
29
					run_fvcom[-P{{CYCLE_FREQ}}D] =>  start_cycle => softlink_forcing  & softlink_forcing_remote => write_run_namelist
30 31

{% if FORECAST and (WIND_ON == 'T' or HEATING_ON == 'T' or PRECIPITATION_ON == 'T' or AIR_PRESSURE_ON == 'T') %}
32 33 34 35 36 37 38 39
					start_cycle => wrf_suite_trigger <wrf::transfer_for_fvcom> 
					wrf_suite_trigger <wrf::transfer_for_fvcom>:succeeded & softlink_forcing_remote => get_wrf_file
					wrf_suite_trigger <wrf::transfer_for_fvcom>:failed | get_wrf_file => write_run_namelist
					wrf_suite_trigger <wrf::transfer_for_fvcom>:succeeded => !adjust_namelist_met
					wrf_suite_trigger <wrf::transfer_for_fvcom>:failed & write_run_namelist => adjust_namelist_met
					(wrf_suite_trigger <wrf::transfer_for_fvcom>:succeeded & write_run_namelist) | adjust_namelist_met => run_fvcom


40
{% elif WIND_ON == 'T' or HEATING_ON == 'T' or PRECIPITATION_ON == 'T' or AIR_PRESSURE_ON == 'T' %}
41
					start_cycle => get_archive_wrf_file => write_run_namelist
42 43
{% endif %}

44

45 46
{% if BOUNDARY_FORCING == 'TIDAL' %}
					start_cycle => generate_tide_forcing => write_run_namelist
47
{% elif BOUNDARY_FORCING == 'CMEMS' and FORECAST %}
48
					start_cycle => CMEMS_download_trigger <download::get_CMEMS_data:finish> => generate_CMEMS_nest => copy_CMEMS_nest_to_remote => write_run_namelist
49 50 51 52

{% elif BOUNDARY_FORCING == 'CMEMS' %}
                                        start_cycle => generate_CMEMS_nest => copy_CMEMS_nest_to_remote => write_run_namelist

53 54 55 56
{% elif BOUNDARY_FORCING == 'NEST_FILE' %}
					start_cycle => nest_suite_trigger<{{NEST_RUN_SUITE}}::transfer_data> => get_nest_file => write_run_namelist
{% endif %}

57
{% if RIVER_MODEL == 'NEURAL_NET' and FORECAST %}
Mike Bedington's avatar
Mike Bedington committed
58
					start_cycle => wrf_archive_trigger <wrf::cycle_finished> & update_river_model & softlink_forcing => generate_rivers 
59

60 61
					wrf_archive_trigger <wrf::cycle_finished>:failed | generate_rivers:failed => write_empty_river_nml
					generate_rivers:succeeded => !write_empty_river_nml
62 63
					generate_rivers:succeeded | write_empty_river_nml => mv_river_files => write_river_number

64
					write_run_namelist => write_river_number => run_fvcom
65 66 67
{% elif RIVER_MODEL == 'NEURAL_NET' %}
					softlink_forcing => generate_rivers => mv_river_files
					mv_river_files & write_run_namelist => write_river_number => run_fvcom
68 69 70 71
{% elif RIVER_MODEL == 'CLIMATOLOGY' %}
					start_cycle => get_river_files => write_run_namelist => write_river_number => run_fvcom
{% endif %}

72
{% if SEDIMENT %}
73
					write_cstms_namelist => run_fvcom
74
{% endif %}
75

76
{% if FORECAST %} 
77
					write_run_namelist => run_fvcom => transfer_data => transfer_data_today => remote_archive => clean_output
78
{% else %}
79
					write_run_namelist => run_fvcom => transfer_data => clean_output
80
{% endif %}
81
					run_fvcom => nan_check
82 83


84
					"""
85
		[[[+P{{CYCLE_FREQ}}D/P{{CYCLE_FREQ}}D]]]
86
			graph = """
87
					run_fvcom[-P{{CYCLE_FREQ}}D] => transfer_restart => run_fvcom
88 89
					"""

90
                [[[+P{{CYCLE_FREQ}}D/P{{CYCLE_FREQ}}D]]]
91 92 93 94 95
                        graph = """
                                        run_fvcom => housekeeping
                                        """


96 97
[runtime]
	[[root]]
98
		env-script = eval $(rose task-env --cycle-offset=P{{CYCLE_FREQ}}D)
99 100
		script = rose task-run --verbose
		[[[job]]]
101
			execution time limit = PT{{EXECUTION_TIME_LIMIT}}H
102
		[[[events]]]
103 104
			mail events = submission timeout, execution timeout, failed
			mail to = {{MAIL_TO}}
105 106
			submission timeout = P1D
		[[[environment]]]
107 108 109 110 111 112 113 114
			START_DATE=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			END_DATE=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			START_DATE_STR=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d_%H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			END_DATE_STR=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d_%H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			START_DAY=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)
			END_DAY=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)

			FORECAST_DAY=$(rose date --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)
115 116 117 118

			GRID_NAME={{GRID_NAME}}
			RIVER_MODEL_PATH={{RIVER_MODEL_PATH}}
			REMOTE_TRANSFER_DIR={{REMOTE_TRANSFER_DIR}}
119
			ARCHIVE_DIR={{ARCHIVE_DIR}}
120
			PLOT_DIR={{PLOT_DIR}}
121
			REMOTE_ARCHIVE_DIR={{REMOTE_ARCHIVE_DIR}}
122 123

			HARMONICS_FILE_PATH={{HARMONICS_FILE_PATH}}
124
			DONOR_RESTART_FILE_PATH={{DONOR_RESTART_FILE_PATH}}
125
			CMEMS_DATA_DIR={{CMEMS_DATA_DIR}}
126
			AMM_7={{AMM_7}}
127 128
			NEST_RUN_SUITE={{NEST_RUN_SUITE}}
			NEST_INTERVAL={{NEST_INTERVAL}}
129
			ADJUST_TIDES={{ADJUST_TIDES}}
130
			NO_NESTS={{NEST_DEPTH}}
131 132 133 134 135 136 137 138 139 140 141

			EXT_TSTEP={{EXT_TSTEP}}
			ISPLIT={{ISPLIT}}
			STARTUP_DMAX={{STARTUP_DMAX}}
			HORIZONTAL_MIXING_COEFFICIENT={{HORIZONTAL_MIXING_COEFFICIENT}}
			VERTICAL_MIXING_COEFFICIENT={{VERTICAL_MIXING_COEFFICIENT}}
			COORDINATES={{COORDINATES}}
			WIND_ON={{WIND_ON}}
			HEATING_ON={{HEATING_ON}}
			PRECIPITATION_ON={{PRECIPITATION_ON}}
			AIR_PRESSURE_ON={{AIR_PRESSURE_ON}}
142 143
			OBC_ON={{OBC_ON}}
			RIVER_TS_SETTING={{RIVER_TS_SETTING}}
144

145 146 147
			TEMP_ACTIVE={{TEMP_ACTIVE}}
			SALT_ACTIVE={{SALT_ACTIVE}}

148 149 150 151 152 153 154 155
			NC_INTERVAL={{NC_INTERVAL}}
			NC_AV_ON={{NC_AV_ON}}
			NC_AV_INTERVAL={{NC_AV_INTERVAL}}

			VELOCITY_OUT={{VELOCITY_OUT}}
			SALT_TEMP_OUT={{SALT_TEMP_OUT}}
			TURBULENCE_OUT={{TURBULENCE_OUT}}
			WIND_OUT={{WIND_OUT}}
156
			HEAT_RAIN_OUT={{HEAT_RAIN_OUT}}
157

158 159
			WRF_FORECAST_FILE_DIR={{WRF_FORECAST_FILE_DIR}}
			WRF_ARCHIVE_DIR={{WRF_ARCHIVE_DIR}}
160
			WRF_ARCHIVE_DATEFMT={{WRF_ARCHIVE_DATEFMT}}
161
			COMMON_FILES_PATH={{COMMON_FILES_PATH}}
162
                        WRF_ARCHIVE_DATESTR=$(rose date --print-format={{WRF_ARCHIVE_DATEFMT}} $CYLC_TASK_CYCLE_POINT)
163
			RESTART_ARCHIVE_DIR={{RESTART_ARCHIVE_DIR}}
164 165 166 167 168 169 170 171

{% if FORECAST %}
                        TODAY_WRF_FILE=/{{WRF_ARCHIVE_DIR}}/today/today_wrfout.nc
{% else %}
			TODAY_WRF_FILE='hindcast'
{% endif %}


172
{% if BOUNDARY_FORCING == 'TIDAL' %}
173
			ELEV_FORCE='T'
174
{% else %}
175
			ELEV_FORCE='F'
176
{% endif %}
177

178

179
{% if BOUNDARY_FORCING in ['CMEMS', 'NEST_FILE'] and NEST_DEPTH == 1 %}
180
                        NEST_ON='T'
181 182
			NEST_TYPE='1'
{% elif BOUNDARY_FORCING in ['CMEMS', 'NEST_FILE'] %}
183
                        NEST_ON='T'
184 185 186 187
                        NEST_TYPE='3'
{% else %}
                        NEST_ON='F'
			NEST_TYPE='1'
188
{% endif %}
189 190


191 192 193 194 195 196
	[[slurm_job]]
		[[[job]]]
			batch system = slurm
			submission polling intervals = PT10S
			execution polling intervals = PT10S, PT1M
		[[[directives]]]
197
			--nodes = {{NODES}}
198 199 200
			--ntasks-per-node=20
			--threads-per-core=1
			--time=24:00:00
201
{% if USE_CETO %}
202 203
		[[[remote]]]
			host = login.ceto.npm.ac.uk
204
			owner = {{REMOTE_USER}}
205
{% endif %}
206 207 208 209 210 211 212 213
	[[slurm_job_1]]
		inherit = slurm_job
		[[[directives]]]
			--nodes = 1
			--ntasks-per-node = 1
			--threads-per-core=1
			--time=24:00:00
	[[remote_job]]
214
{% if USE_CETO %}
215 216
		[[[remote]]]
			host = login.ceto.npm.ac.uk
217
			owner = {{REMOTE_USER}}
218
{% endif %}
219 220 221 222


	[[write_run_namelist]]
		inherit = slurm_job_1
223
{% if SEDIMENT %}
224
	[[write_run_cstms]]
225 226 227 228 229 230
{% endif %}

	[[check_output_dir_len]]
                script = """
                        echo "Need to check suite name doesn't take the fvcom output dir over the max length"
                        """
231 232 233

	[[copy_common_to_remote]]
		inherit = remote_job
234 235
	[[softlink_forcing]]
	[[softlink_forcing_remote]]
236
	        inherit = slurm_job_1
237 238 239 240 241 242 243 244 245 246 247 248 249 250
	
	[[wrf_suite_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
		[[[job]]]
			execution retry delays = 3*PT15M
	[[get_wrf_file]]
		inherit = slurm_job_1


	[[generate_tide_forcing]]
		inherit = slurm_job_1
251
    	[[CMEMS_download_trigger]]
252 253 254 255 256 257
        script =""
        [[[suite state polling]]]
            interval = PT1M
            max-polls = 1440
        [[[job]]]
            execution retry delays = 3*PT15M
258
	[[generate_CMEMS_nest]]
Mike Bedington's avatar
Mike Bedington committed
259 260 261 262
	[[copy_CMEMS_nest_to_remote]]
		inherit = remote_job
		script = "cp /pml${REMOTE_TRANSFER_DIR}/boundary_nest.nc ${ROSE_DATAC}/${GRID_NAME}_nest_0001.nc"

263 264 265 266 267 268 269 270 271 272 273 274 275 276
	[[nest_suite_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
	[[get_nest_file]]
		inherit = slurm_job_1

	[[wrf_archive_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
		[[[job]]]
277
			execution retry delays = 3*PT15M	
278 279 280 281
	[[get_archive_wrf_file]]
		inherit = remote_job
	[[convert_wrf_file]]
		inherit = slurm_job_1
282 283 284 285 286 287 288 289 290
	[[adjust_namelist_met]]
		inherit = remote_job
                script = """
                        sed -i "s|.*AIRPRESSURE_ON.*|AIRPRESSURE_ON=F,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
                        sed -i "s|.*HEATING_ON.*|HEATING_ON=F,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
                        sed -i "s|.*RECIPITATION_ON.*|PRECIPITATION_ON=F,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			sed -i "s|.*WIND_ON.*|WIND_ON=T,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			"""

291

292 293 294 295 296 297 298 299 300 301 302 303
	[[update_river_model]]
	[[generate_rivers]]
	[[get_river_files]]
		inherit = slurm_job_1
	[[mv_river_files]]
		inherit = remote_job
	[[write_river_number]]
		inherit = slurm_job_1
		script = """
				NO_RIVERS=$(grep "&NML_RIVER" ${ROSE_DATAC}/${GRID_NAME}_riv.nml | wc -l);
				sed -i "s|.*RIVER_NUMBER.*| RIVER_NUMBER  = ${NO_RIVERS},|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
				"""
304 305
	[[write_empty_river_nml]]
		script = """
306
			touch ${GRID_NAME}_riv.nml; cp ${GRID_NAME}_riv.nml /${REMOTE_TRANSFER_DIR}/
307
			"""
308 309 310

	[[run_fvcom]]
		inherit = slurm_job
311
	[[adjust_namelist]]
312 313 314 315
		inherit = remote_job
		script = """
			sed -i "s|.*STARTUP_UV_TYPE.*|STARTUP_UV_TYPE   = 'default',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			sed -i "s|.*STARTUP_TURB_TYPE.*|STARTUP_TURB_TYPE = 'default',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
316
			sed -i "s|.*STARTUP_TS_TYPE.*|STARTUP_TS_TYPE   = 'set values',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
317
{% if SEDIMENT %}
318
			sed -i "s|.*SED_HOT_START.*|SED_HOT_START = F|" ${ROSE_DATAC}/cstms_sediment.inp
319
{% endif %}
320
			"""
321 322
	[[generate_CMEMS_start]]
	[[mv_start_file]]
323
		inherit = remote_job
324
	[[transfer_data]]
325
		inherit = remote_job
326 327
	[[transfer_data_today]]
		inherit = slurm_job_1
328 329 330 331 332
	[[start_cycle]]
		script = """
			echo "starting forecast cycle"
			"""
	[[transfer_restart]]
333
		inherit = remote_job
334
		script = """
335
			ln -sf ${ROSE_DATACP{{CYCLE_FREQ}}D}/output/${GRID_NAME}_restart_0001.nc ${ROSE_DATAC}/${GRID_NAME}_restart_0001.nc
336
			mkdir -p /pml${RESTART_ARCHIVE_DIR}/${FORECAST_DAY}
337
			cp ${ROSE_DATACP{{CYCLE_FREQ}}D}/output/${GRID_NAME}_restart_0001.nc /pml${RESTART_ARCHIVE_DIR}/${FORECAST_DAY}/
338 339 340 341 342
			"""
	[[transfer_archive_restart]]
		inherit = remote_job
		script = """
			cp /pml${RESTART_ARCHIVE_DIR}/${FORECAST_DAY}/${GRID_NAME}_restart_0001.nc ${ROSE_DATAC}
343
			"""
344

345 346
	[[nan_check]]
		inherit = remote_job
347 348 349 350 351 352 353
	[[remote_archive]]
		inherit = remote_job
	[[clean_output]]
		inherit = remote_job
		script = """
			rm ${ROSE_DATAC}/output/${GRID_NAME}_0001.nc
			"""
354 355 356 357 358 359
	[[housekeeping]]
		inherit = remote_job
                script = """
                        rm ${ROSE_DATACP2D}/output/${GRID_NAME}_restart_0001.nc
                        """