suite.rc 10.3 KB
Newer Older
1 2 3 4 5 6
#!jinja2
[cylc]
	UTC mode = True # Ignore DST
	abort if any task fails = False

[scheduling]
7
	initial cycle point = {{INITIAL_START_DATE}}
8
{% if FINAL_CYCLE_POINT != 'NONE' %}
9
        final cycle point = {{FINAL_CYCLE_POINT}}
10
{% endif %}
11 12 13 14 15 16
	[[special tasks]]
		clock-trigger = start_cycle(PT0M)
	[[dependencies]]
		[[[R1]]]
			graph = """
					copy_common_to_remote => softlink_forcing_remote
17
{% if COLD_START and BOUNDARY_FORCING == 'CMEMS' %}
18 19 20
					write_run_namelist => adjust_namelist & generate_CMEMS_start => mv_start_file => run_fvcom
{% elif COLD_START and BOUNDARY_FORCING == 'NEST_FILE' %}
					write_run_namelist => adjust_namelist & generate_nest_start => mv_start_file => run_fvcom
21
{% endif %}
22 23 24 25 26 27
					"""
		[[[P1D]]]
			graph = """
					run_fvcom[-P1D] =>  start_cycle => softlink_forcing  & softlink_forcing_remote => write_run_namelist

{% if FORECAST and (WIND_ON == 'T' or HEATING_ON == 'T' or PRECIPITATION_ON == 'T' or AIR_PRESSURE_ON == 'T') %}
28 29 30 31 32 33 34 35
					start_cycle => wrf_suite_trigger <wrf::transfer_for_fvcom> 
					wrf_suite_trigger <wrf::transfer_for_fvcom>:succeeded & softlink_forcing_remote => get_wrf_file
					wrf_suite_trigger <wrf::transfer_for_fvcom>:failed | get_wrf_file => write_run_namelist
					wrf_suite_trigger <wrf::transfer_for_fvcom>:succeeded => !adjust_namelist_met
					wrf_suite_trigger <wrf::transfer_for_fvcom>:failed & write_run_namelist => adjust_namelist_met
					(wrf_suite_trigger <wrf::transfer_for_fvcom>:succeeded & write_run_namelist) | adjust_namelist_met => run_fvcom


36 37
{% elif WIND_ON == 'T' or HEATING_ON == 'T' or PRECIPITATION_ON == 'T' or AIR_PRESSURE_ON == 'T' %}
					start_cycle => get_archive_wrf_file => convert_wrf_file => write_run_namelist
38 39
{% endif %}

40

41 42
{% if BOUNDARY_FORCING == 'TIDAL' %}
					start_cycle => generate_tide_forcing => write_run_namelist
43
{% elif BOUNDARY_FORCING == 'CMEMS' and FORECAST %}
44
					start_cycle => CMEMS_download_trigger <download::get_CMEMS_data:finish> => generate_CMEMS_nest => copy_CMEMS_nest_to_remote => write_run_namelist
45 46 47 48

{% elif BOUNDARY_FORCING == 'CMEMS' %}
                                        start_cycle => generate_CMEMS_nest => copy_CMEMS_nest_to_remote => write_run_namelist

49 50 51 52
{% elif BOUNDARY_FORCING == 'NEST_FILE' %}
					start_cycle => nest_suite_trigger<{{NEST_RUN_SUITE}}::transfer_data> => get_nest_file => write_run_namelist
{% endif %}

53
{% if RIVER_MODEL == 'NEURAL_NET' and FORECAST %}
Mike Bedington's avatar
Mike Bedington committed
54
					start_cycle => wrf_archive_trigger <wrf::cycle_finished> & update_river_model & softlink_forcing => generate_rivers 
55

56 57
					wrf_archive_trigger <wrf::cycle_finished>:failed | generate_rivers:failed => write_empty_river_nml
					generate_rivers:succeeded => !write_empty_river_nml
58 59
					generate_rivers:succeeded | write_empty_river_nml => mv_river_files => write_river_number

60
					write_run_namelist => write_river_number => run_fvcom
61 62 63
{% elif RIVER_MODEL == 'NEURAL_NET' %}
					softlink_forcing => generate_rivers => mv_river_files
					mv_river_files & write_run_namelist => write_river_number => run_fvcom
64 65 66 67
{% elif RIVER_MODEL == 'CLIMATOLOGY' %}
					start_cycle => get_river_files => write_run_namelist => write_river_number => run_fvcom
{% endif %}

68
{% if SEDIMENT %}
69
					write_cstms_namelist => run_fvcom
70
{% endif %}
71

72
{% if FORECAST %} 
73
					write_run_namelist => run_fvcom => transfer_data => transfer_data_today
74
{% else %}
75
					write_run_namelist => run_fvcom => transfer_data
76
{% endif %}
77 78
					run_fvcom => nan_check 
					transfer_data => plot_surf_vars
79 80


81 82 83 84 85 86 87 88 89 90 91 92 93
					"""
		[[[+P1D/P1D]]]
			graph = """
					run_fvcom[-P1D] => transfer_restart => run_fvcom
					"""

[runtime]
	[[root]]
		env-script = eval $(rose task-env --cycle-offset=P1D)
		script = rose task-run --verbose
		[[[job]]]
			execution time limit = PT3H
		[[[events]]]
94 95
			mail events = submission timeout, execution timeout, failed
			mail to = {{MAIL_TO}}
96 97
			submission timeout = P1D
		[[[environment]]]
98 99 100 101 102 103 104 105
			START_DATE=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			END_DATE=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			START_DATE_STR=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d_%H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			END_DATE_STR=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d_%H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			START_DAY=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)
			END_DAY=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)

			FORECAST_DAY=$(rose date --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)
106 107 108 109

			GRID_NAME={{GRID_NAME}}
			RIVER_MODEL_PATH={{RIVER_MODEL_PATH}}
			REMOTE_TRANSFER_DIR={{REMOTE_TRANSFER_DIR}}
110
			ARCHIVE_DIR={{ARCHIVE_DIR}}
111
			PLOT_DIR={{PLOT_DIR}}
112 113

			HARMONICS_FILE_PATH={{HARMONICS_FILE_PATH}}
114
			DONOR_RESTART_FILE_PATH={{DONOR_RESTART_FILE_PATH}}
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129
			CMEMS_DATA_DIR={{CMEMS_DATA_DIR}}
			NEST_RUN_SUITE={{NEST_RUN_SUITE}}
			NEST_INTERVAL={{NEST_INTERVAL}}

			EXT_TSTEP={{EXT_TSTEP}}
			ISPLIT={{ISPLIT}}
			STARTUP_DMAX={{STARTUP_DMAX}}
			HORIZONTAL_MIXING_COEFFICIENT={{HORIZONTAL_MIXING_COEFFICIENT}}
			VERTICAL_MIXING_COEFFICIENT={{VERTICAL_MIXING_COEFFICIENT}}
			COORDINATES={{COORDINATES}}
			WIND_ON={{WIND_ON}}
			HEATING_ON={{HEATING_ON}}
			PRECIPITATION_ON={{PRECIPITATION_ON}}
			AIR_PRESSURE_ON={{AIR_PRESSURE_ON}}

130 131 132
			TEMP_ACTIVE={{TEMP_ACTIVE}}
			SALT_ACTIVE={{SALT_ACTIVE}}

133 134 135 136 137 138 139 140
			NC_INTERVAL={{NC_INTERVAL}}
			NC_AV_ON={{NC_AV_ON}}
			NC_AV_INTERVAL={{NC_AV_INTERVAL}}

			VELOCITY_OUT={{VELOCITY_OUT}}
			SALT_TEMP_OUT={{SALT_TEMP_OUT}}
			TURBULENCE_OUT={{TURBULENCE_OUT}}
			WIND_OUT={{WIND_OUT}}
141
			HEAT_RAIN_OUT={{HEAT_RAIN_OUT}}
142

143 144
			WRF_FORECAST_FILE_DIR={{WRF_FORECAST_FILE_DIR}}
			WRF_ARCHIVE_DIR={{WRF_ARCHIVE_DIR}}
145
			WRF_ARCHIVE_DATEFMT={{WRF_ARCHIVE_DATEFMT}}
146
			COMMON_FILES_PATH={{COMMON_FILES_PATH}}
147 148 149 150 151 152 153 154 155
                        WRF_ARCHIVE_DATESTR=$(rose date --print-format={{WRF_ARCHIVE_DATEFMT}} $CYLC_TASK_CYCLE_POINT)

{% if FORECAST %}
                        TODAY_WRF_FILE=/{{WRF_ARCHIVE_DIR}}/today/today_wrfout.nc
{% else %}
			TODAY_WRF_FILE='hindcast'
{% endif %}


156
{% if BOUNDARY_FORCING == 'TIDAL' %}
157
			ELEV_FORCE='T'
158
{% else %}
159
			ELEV_FORCE='F'
160
{% endif %}
161

162

163
{% if BOUNDARY_FORCING in ['CMEMS', 'NEST_FILE'] %}
164 165
                        NEST_ON='T'
			NEST_TYPE='3'
166
{% else %}
167 168
                        NEST_ON='T'
			NEST_TYPE='3'
169
{% endif %}
170 171


172 173 174 175 176 177
	[[slurm_job]]
		[[[job]]]
			batch system = slurm
			submission polling intervals = PT10S
			execution polling intervals = PT10S, PT1M
		[[[directives]]]
178
			--nodes = {{NODES}}
179 180 181
			--ntasks-per-node=20
			--threads-per-core=1
			--time=24:00:00
182
{% if USE_CETO %}
183 184
		[[[remote]]]
			host = login.ceto.npm.ac.uk
185
			owner = {{REMOTE_USER}}
186
{% endif %}
187 188 189 190 191 192 193 194
	[[slurm_job_1]]
		inherit = slurm_job
		[[[directives]]]
			--nodes = 1
			--ntasks-per-node = 1
			--threads-per-core=1
			--time=24:00:00
	[[remote_job]]
195
{% if USE_CETO %}
196 197
		[[[remote]]]
			host = login.ceto.npm.ac.uk
198
			owner = {{REMOTE_USER}}
199
{% endif %}
200 201 202 203


	[[write_run_namelist]]
		inherit = slurm_job_1
204
{% if SEDIMENT %}
205
	[[write_run_cstms]]
206 207 208 209 210 211
{% endif %}

	[[check_output_dir_len]]
                script = """
                        echo "Need to check suite name doesn't take the fvcom output dir over the max length"
                        """
212 213 214

	[[copy_common_to_remote]]
		inherit = remote_job
215 216
	[[softlink_forcing]]
	[[softlink_forcing_remote]]
217
	        inherit = slurm_job_1
218 219 220 221 222 223 224 225 226 227 228 229 230 231
	
	[[wrf_suite_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
		[[[job]]]
			execution retry delays = 3*PT15M
	[[get_wrf_file]]
		inherit = slurm_job_1


	[[generate_tide_forcing]]
		inherit = slurm_job_1
232
    	[[CMEMS_download_trigger]]
233 234 235 236 237 238
        script =""
        [[[suite state polling]]]
            interval = PT1M
            max-polls = 1440
        [[[job]]]
            execution retry delays = 3*PT15M
239
	[[generate_CMEMS_nest]]
Mike Bedington's avatar
Mike Bedington committed
240 241 242 243
	[[copy_CMEMS_nest_to_remote]]
		inherit = remote_job
		script = "cp /pml${REMOTE_TRANSFER_DIR}/boundary_nest.nc ${ROSE_DATAC}/${GRID_NAME}_nest_0001.nc"

244 245 246 247 248 249 250 251 252 253 254 255 256 257
	[[nest_suite_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
	[[get_nest_file]]
		inherit = slurm_job_1

	[[wrf_archive_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
		[[[job]]]
258
			execution retry delays = 3*PT15M	
259 260 261 262
	[[get_archive_wrf_file]]
		inherit = remote_job
	[[convert_wrf_file]]
		inherit = slurm_job_1
263 264 265 266 267 268 269 270 271
	[[adjust_namelist_met]]
		inherit = remote_job
                script = """
                        sed -i "s|.*AIRPRESSURE_ON.*|AIRPRESSURE_ON=F,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
                        sed -i "s|.*HEATING_ON.*|HEATING_ON=F,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
                        sed -i "s|.*RECIPITATION_ON.*|PRECIPITATION_ON=F,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			sed -i "s|.*WIND_ON.*|WIND_ON=T,|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			"""

272

273 274 275 276 277 278 279 280 281 282 283 284
	[[update_river_model]]
	[[generate_rivers]]
	[[get_river_files]]
		inherit = slurm_job_1
	[[mv_river_files]]
		inherit = remote_job
	[[write_river_number]]
		inherit = slurm_job_1
		script = """
				NO_RIVERS=$(grep "&NML_RIVER" ${ROSE_DATAC}/${GRID_NAME}_riv.nml | wc -l);
				sed -i "s|.*RIVER_NUMBER.*| RIVER_NUMBER  = ${NO_RIVERS},|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
				"""
285 286
	[[write_empty_river_nml]]
		script = """
287
			touch ${GRID_NAME}_riv.nml; cp ${GRID_NAME}_riv.nml /${REMOTE_TRANSFER_DIR}/
288
			"""
289 290 291

	[[run_fvcom]]
		inherit = slurm_job
292
	[[adjust_namelist]]
293 294 295 296
		inherit = remote_job
		script = """
			sed -i "s|.*STARTUP_UV_TYPE.*|STARTUP_UV_TYPE   = 'default',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			sed -i "s|.*STARTUP_TURB_TYPE.*|STARTUP_TURB_TYPE = 'default',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
297
			sed -i "s|.*STARTUP_TS_TYPE.*|STARTUP_TS_TYPE   = 'set values',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
298
{% if SEDIMENT %}
299
			sed -i "s|.*SED_HOT_START.*|SED_HOT_START = F|" ${ROSE_DATAC}/cstms_sediment.inp
300
{% endif %}
301
			"""
302 303
	[[generate_CMEMS_start]]
	[[mv_start_file]]
304
		inherit = remote_job
305 306
	[[transfer_data]]
		inherit = slurm_job_1
307 308
	[[transfer_data_today]]
		inherit = slurm_job_1
309 310 311 312 313 314 315 316 317
	[[start_cycle]]
		script = """
			echo "starting forecast cycle"
			"""
	[[transfer_restart]]
		inherit = slurm_job
		script = """
			ln -s ${ROSE_DATACP1D}/output/${GRID_NAME}_restart_0001.nc ${ROSE_DATAC}/${GRID_NAME}_restart_0001.nc
			"""
318 319
	[[nan_check]]
		inherit = remote_job
320
	[[plot_surf_vars]]