suite.rc 10.5 KB
Newer Older
1 2 3 4 5 6
#!jinja2
[cylc]
	UTC mode = True # Ignore DST
	abort if any task fails = False

[scheduling]
7
	initial cycle point = {{INITIAL_START_DATE}}
8
{% if FINAL_CYCLE_POINT != 'NONE' %}
9
        final cycle point = {{FINAL_CYCLE_POINT}}
10
{% endif %}
11 12 13 14 15 16
	[[special tasks]]
		clock-trigger = start_cycle(PT0M)
	[[dependencies]]
		[[[R1]]]
			graph = """
					copy_common_to_remote => softlink_forcing_remote
17
{% if COLD_START and BOUNDARY_FORCING == 'CMEMS' %}
18 19 20
					write_run_namelist => adjust_namelist & generate_CMEMS_start => mv_start_file => run_fvcom
{% elif COLD_START and BOUNDARY_FORCING == 'NEST_FILE' %}
					write_run_namelist => adjust_namelist & generate_nest_start => mv_start_file => run_fvcom
21 22
{% else %}
					transfer_restart => run_fvcom
23
{% endif %}
24 25 26 27 28 29
					"""
		[[[P1D]]]
			graph = """
					run_fvcom[-P1D] =>  start_cycle => softlink_forcing  & softlink_forcing_remote => write_run_namelist

{% if FORECAST and (WIND_ON == 'T' or HEATING_ON == 'T' or PRECIPITATION_ON == 'T' or AIR_PRESSURE_ON == 'T') %}
30 31 32 33 34 35 36 37
					start_cycle => wrf_suite_trigger <wrf::transfer_for_fvcom> 
					wrf_suite_trigger <wrf::transfer_for_fvcom>:succeeded & softlink_forcing_remote => get_wrf_file
					wrf_suite_trigger <wrf::transfer_for_fvcom>:failed | get_wrf_file => write_run_namelist
					wrf_suite_trigger <wrf::transfer_for_fvcom>:succeeded => !adjust_namelist_met
					wrf_suite_trigger <wrf::transfer_for_fvcom>:failed & write_run_namelist => adjust_namelist_met
					(wrf_suite_trigger <wrf::transfer_for_fvcom>:succeeded & write_run_namelist) | adjust_namelist_met => run_fvcom


38 39
{% elif WIND_ON == 'T' or HEATING_ON == 'T' or PRECIPITATION_ON == 'T' or AIR_PRESSURE_ON == 'T' %}
					start_cycle => get_archive_wrf_file => convert_wrf_file => write_run_namelist
40 41
{% endif %}

42

43 44
{% if BOUNDARY_FORCING == 'TIDAL' %}
					start_cycle => generate_tide_forcing => write_run_namelist
45
{% elif BOUNDARY_FORCING == 'CMEMS' and FORECAST %}
46
					start_cycle => CMEMS_download_trigger <download::get_CMEMS_data:finish> => generate_CMEMS_nest => copy_CMEMS_nest_to_remote => write_run_namelist
47 48 49 50

{% elif BOUNDARY_FORCING == 'CMEMS' %}
                                        start_cycle => generate_CMEMS_nest => copy_CMEMS_nest_to_remote => write_run_namelist

51 52 53 54
{% elif BOUNDARY_FORCING == 'NEST_FILE' %}
					start_cycle => nest_suite_trigger<{{NEST_RUN_SUITE}}::transfer_data> => get_nest_file => write_run_namelist
{% endif %}

55
{% if RIVER_MODEL == 'NEURAL_NET' and FORECAST %}
Mike Bedington's avatar
Mike Bedington committed
56
					start_cycle => wrf_archive_trigger <wrf::cycle_finished> & update_river_model & softlink_forcing => generate_rivers 
57

58 59
					wrf_archive_trigger <wrf::cycle_finished>:failed | generate_rivers:failed => write_empty_river_nml
					generate_rivers:succeeded => !write_empty_river_nml
60 61
					generate_rivers:succeeded | write_empty_river_nml => mv_river_files => write_river_number

62
					write_run_namelist => write_river_number => run_fvcom
63 64 65
{% elif RIVER_MODEL == 'NEURAL_NET' %}
					softlink_forcing => generate_rivers => mv_river_files
					mv_river_files & write_run_namelist => write_river_number => run_fvcom
66 67 68 69
{% elif RIVER_MODEL == 'CLIMATOLOGY' %}
					start_cycle => get_river_files => write_run_namelist => write_river_number => run_fvcom
{% endif %}

70
{% if SEDIMENT %}
71
					write_cstms_namelist => run_fvcom
72
{% endif %}
73

74
{% if FORECAST %} 
75
					write_run_namelist => run_fvcom => transfer_data => transfer_data_today => remote_archive => clean_output
76
{% else %}
77
					write_run_namelist => run_fvcom => transfer_data => clean_output
78
{% endif %}
79
					run_fvcom => nan_check 
80 81


82 83 84 85 86 87 88 89 90 91 92 93 94
					"""
		[[[+P1D/P1D]]]
			graph = """
					run_fvcom[-P1D] => transfer_restart => run_fvcom
					"""

[runtime]
	[[root]]
		env-script = eval $(rose task-env --cycle-offset=P1D)
		script = rose task-run --verbose
		[[[job]]]
			execution time limit = PT3H
		[[[events]]]
95 96
			mail events = submission timeout, execution timeout, failed
			mail to = {{MAIL_TO}}
97 98
			submission timeout = P1D
		[[[environment]]]
99 100 101 102 103 104 105 106
			START_DATE=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			END_DATE=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			START_DATE_STR=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d_%H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			END_DATE_STR=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d_%H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			START_DAY=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)
			END_DAY=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)

			FORECAST_DAY=$(rose date --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)
107 108 109 110

			GRID_NAME={{GRID_NAME}}
			RIVER_MODEL_PATH={{RIVER_MODEL_PATH}}
			REMOTE_TRANSFER_DIR={{REMOTE_TRANSFER_DIR}}
111
			ARCHIVE_DIR={{ARCHIVE_DIR}}
112
			PLOT_DIR={{PLOT_DIR}}
113
			REMOTE_ARCHIVE_DIR={{REMOTE_ARCHIVE_DIR}}
114 115

			HARMONICS_FILE_PATH={{HARMONICS_FILE_PATH}}
116
			DONOR_RESTART_FILE_PATH={{DONOR_RESTART_FILE_PATH}}
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
			CMEMS_DATA_DIR={{CMEMS_DATA_DIR}}
			NEST_RUN_SUITE={{NEST_RUN_SUITE}}
			NEST_INTERVAL={{NEST_INTERVAL}}

			EXT_TSTEP={{EXT_TSTEP}}
			ISPLIT={{ISPLIT}}
			STARTUP_DMAX={{STARTUP_DMAX}}
			HORIZONTAL_MIXING_COEFFICIENT={{HORIZONTAL_MIXING_COEFFICIENT}}
			VERTICAL_MIXING_COEFFICIENT={{VERTICAL_MIXING_COEFFICIENT}}
			COORDINATES={{COORDINATES}}
			WIND_ON={{WIND_ON}}
			HEATING_ON={{HEATING_ON}}
			PRECIPITATION_ON={{PRECIPITATION_ON}}
			AIR_PRESSURE_ON={{AIR_PRESSURE_ON}}

132 133 134
			TEMP_ACTIVE={{TEMP_ACTIVE}}
			SALT_ACTIVE={{SALT_ACTIVE}}

135 136 137 138 139 140 141 142
			NC_INTERVAL={{NC_INTERVAL}}
			NC_AV_ON={{NC_AV_ON}}
			NC_AV_INTERVAL={{NC_AV_INTERVAL}}

			VELOCITY_OUT={{VELOCITY_OUT}}
			SALT_TEMP_OUT={{SALT_TEMP_OUT}}
			TURBULENCE_OUT={{TURBULENCE_OUT}}
			WIND_OUT={{WIND_OUT}}
143
			HEAT_RAIN_OUT={{HEAT_RAIN_OUT}}
144

145 146
			WRF_FORECAST_FILE_DIR={{WRF_FORECAST_FILE_DIR}}
			WRF_ARCHIVE_DIR={{WRF_ARCHIVE_DIR}}
147
			WRF_ARCHIVE_DATEFMT={{WRF_ARCHIVE_DATEFMT}}
148
			COMMON_FILES_PATH={{COMMON_FILES_PATH}}
149 150 151 152 153 154 155 156 157
                        WRF_ARCHIVE_DATESTR=$(rose date --print-format={{WRF_ARCHIVE_DATEFMT}} $CYLC_TASK_CYCLE_POINT)

{% if FORECAST %}
                        TODAY_WRF_FILE=/{{WRF_ARCHIVE_DIR}}/today/today_wrfout.nc
{% else %}
			TODAY_WRF_FILE='hindcast'
{% endif %}


158
{% if BOUNDARY_FORCING == 'TIDAL' %}
159
			ELEV_FORCE='T'
160
{% else %}
161
			ELEV_FORCE='F'
162
{% endif %}
163

164

165
{% if BOUNDARY_FORCING in ['CMEMS', 'NEST_FILE'] %}
166 167
                        NEST_ON='T'
			NEST_TYPE='3'
168
{% else %}
169 170
                        NEST_ON='T'
			NEST_TYPE='3'
171
{% endif %}
172 173


174 175 176 177 178 179
	[[slurm_job]]
		[[[job]]]
			batch system = slurm
			submission polling intervals = PT10S
			execution polling intervals = PT10S, PT1M
		[[[directives]]]
180
			--nodes = {{NODES}}
181 182 183
			--ntasks-per-node=20
			--threads-per-core=1
			--time=24:00:00
184
{% if USE_CETO %}
185 186
		[[[remote]]]
			host = login.ceto.npm.ac.uk
187
			owner = {{REMOTE_USER}}
188
{% endif %}
189 190 191 192 193 194 195 196
	[[slurm_job_1]]
		inherit = slurm_job
		[[[directives]]]
			--nodes = 1
			--ntasks-per-node = 1
			--threads-per-core=1
			--time=24:00:00
	[[remote_job]]
197
{% if USE_CETO %}
198 199
		[[[remote]]]
			host = login.ceto.npm.ac.uk
200
			owner = {{REMOTE_USER}}
201
{% endif %}
202 203 204 205


	[[write_run_namelist]]
		inherit = slurm_job_1
206
{% if SEDIMENT %}
207
	[[write_run_cstms]]
208 209 210 211 212 213
{% endif %}

	[[check_output_dir_len]]
                script = """
                        echo "Need to check suite name doesn't take the fvcom output dir over the max length"
                        """
214 215 216

	[[copy_common_to_remote]]
		inherit = remote_job
217 218
	[[softlink_forcing]]
	[[softlink_forcing_remote]]
219
	        inherit = slurm_job_1
220 221 222 223 224 225 226 227 228 229 230 231 232 233
	
	[[wrf_suite_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
		[[[job]]]
			execution retry delays = 3*PT15M
	[[get_wrf_file]]
		inherit = slurm_job_1


	[[generate_tide_forcing]]
		inherit = slurm_job_1
234
    	[[CMEMS_download_trigger]]
235 236 237 238 239 240
        script =""
        [[[suite state polling]]]
            interval = PT1M
            max-polls = 1440
        [[[job]]]
            execution retry delays = 3*PT15M
241
	[[generate_CMEMS_nest]]
Mike Bedington's avatar
Mike Bedington committed
242 243 244 245
	[[copy_CMEMS_nest_to_remote]]
		inherit = remote_job
		script = "cp /pml${REMOTE_TRANSFER_DIR}/boundary_nest.nc ${ROSE_DATAC}/${GRID_NAME}_nest_0001.nc"

246 247 248 249 250 251 252 253 254 255 256 257 258 259
	[[nest_suite_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
	[[get_nest_file]]
		inherit = slurm_job_1

	[[wrf_archive_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
		[[[job]]]
260
			execution retry delays = 3*PT15M	
261 262 263 264
	[[get_archive_wrf_file]]
		inherit = remote_job
	[[convert_wrf_file]]
		inherit = slurm_job_1
265 266 267 268 269 270 271 272 273
	[[adjust_namelist_met]]
		inherit = remote_job
                script = """
                        sed -i "s|.*AIRPRESSURE_ON.*|AIRPRESSURE_ON=F,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
                        sed -i "s|.*HEATING_ON.*|HEATING_ON=F,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
                        sed -i "s|.*RECIPITATION_ON.*|PRECIPITATION_ON=F,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			sed -i "s|.*WIND_ON.*|WIND_ON=T,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			"""

274

275 276 277 278 279 280 281 282 283 284 285 286
	[[update_river_model]]
	[[generate_rivers]]
	[[get_river_files]]
		inherit = slurm_job_1
	[[mv_river_files]]
		inherit = remote_job
	[[write_river_number]]
		inherit = slurm_job_1
		script = """
				NO_RIVERS=$(grep "&NML_RIVER" ${ROSE_DATAC}/${GRID_NAME}_riv.nml | wc -l);
				sed -i "s|.*RIVER_NUMBER.*| RIVER_NUMBER  = ${NO_RIVERS},|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
				"""
287 288
	[[write_empty_river_nml]]
		script = """
289
			touch ${GRID_NAME}_riv.nml; cp ${GRID_NAME}_riv.nml /${REMOTE_TRANSFER_DIR}/
290
			"""
291 292 293

	[[run_fvcom]]
		inherit = slurm_job
294
	[[adjust_namelist]]
295 296 297 298
		inherit = remote_job
		script = """
			sed -i "s|.*STARTUP_UV_TYPE.*|STARTUP_UV_TYPE   = 'default',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			sed -i "s|.*STARTUP_TURB_TYPE.*|STARTUP_TURB_TYPE = 'default',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
299
			sed -i "s|.*STARTUP_TS_TYPE.*|STARTUP_TS_TYPE   = 'set values',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
300
{% if SEDIMENT %}
301
			sed -i "s|.*SED_HOT_START.*|SED_HOT_START = F|" ${ROSE_DATAC}/cstms_sediment.inp
302
{% endif %}
303
			"""
304 305
	[[generate_CMEMS_start]]
	[[mv_start_file]]
306
		inherit = remote_job
307 308
	[[transfer_data]]
		inherit = slurm_job_1
309 310
	[[transfer_data_today]]
		inherit = slurm_job_1
311 312 313 314 315
	[[start_cycle]]
		script = """
			echo "starting forecast cycle"
			"""
	[[transfer_restart]]
316
		inherit = slurm_job_1
317
		script = """
318
			ln -sf ${ROSE_DATACP1D}/output/${GRID_NAME}_restart_0001.nc ${ROSE_DATAC}/${GRID_NAME}_restart_0001.nc
319
			"""
320 321
	[[nan_check]]
		inherit = remote_job
322 323 324 325 326 327 328
	[[remote_archive]]
		inherit = remote_job
	[[clean_output]]
		inherit = remote_job
		script = """
			rm ${ROSE_DATAC}/output/${GRID_NAME}_0001.nc
			"""