suite.rc 9.49 KB
Newer Older
1 2 3 4 5 6
#!jinja2
[cylc]
	UTC mode = True # Ignore DST
	abort if any task fails = False

[scheduling]
7
	initial cycle point = {{INITIAL_START_DATE}}
8
{% if FINAL_CYCLE_POINT != 'NONE' %}
9
        final cycle point = {{FINAL_CYCLE_POINT}}
10
{% endif %}
11 12 13 14 15 16
	[[special tasks]]
		clock-trigger = start_cycle(PT0M)
	[[dependencies]]
		[[[R1]]]
			graph = """
					copy_common_to_remote => softlink_forcing_remote
17
{% if COLD_START and BOUNDARY_FORCING == 'CMEMS' %}
18 19 20
					write_run_namelist => adjust_namelist & generate_CMEMS_start => mv_start_file => run_fvcom
{% elif COLD_START and BOUNDARY_FORCING == 'NEST_FILE' %}
					write_run_namelist => adjust_namelist & generate_nest_start => mv_start_file => run_fvcom
21
{% endif %}
22 23 24 25 26 27
					"""
		[[[P1D]]]
			graph = """
					run_fvcom[-P1D] =>  start_cycle => softlink_forcing  & softlink_forcing_remote => write_run_namelist

{% if FORECAST and (WIND_ON == 'T' or HEATING_ON == 'T' or PRECIPITATION_ON == 'T' or AIR_PRESSURE_ON == 'T') %}
28 29 30
					start_cycle => wrf_suite_trigger <wrf::transfer_for_fvcom> => get_wrf_file => write_run_namelist
{% elif WIND_ON == 'T' or HEATING_ON == 'T' or PRECIPITATION_ON == 'T' or AIR_PRESSURE_ON == 'T' %}
					start_cycle => get_archive_wrf_file => convert_wrf_file => write_run_namelist
31 32
{% endif %}

33

34 35
{% if BOUNDARY_FORCING == 'TIDAL' %}
					start_cycle => generate_tide_forcing => write_run_namelist
36
{% elif BOUNDARY_FORCING == 'CMEMS' and FORECAST %}
37
					start_cycle => CMEMS_download_trigger <download::get_CMEMS_data:finish> => generate_CMEMS_nest => copy_CMEMS_nest_to_remote => write_run_namelist
38 39 40 41

{% elif BOUNDARY_FORCING == 'CMEMS' %}
                                        start_cycle => generate_CMEMS_nest => copy_CMEMS_nest_to_remote => write_run_namelist

42 43 44 45
{% elif BOUNDARY_FORCING == 'NEST_FILE' %}
					start_cycle => nest_suite_trigger<{{NEST_RUN_SUITE}}::transfer_data> => get_nest_file => write_run_namelist
{% endif %}

46
{% if RIVER_MODEL == 'NEURAL_NET' and FORECAST %}
Mike Bedington's avatar
Mike Bedington committed
47
					start_cycle => wrf_archive_trigger <wrf::cycle_finished> & update_river_model & softlink_forcing => generate_rivers 
48 49 50 51 52

					generate_rivers:failed => trigger_archive_wrf & write_empty_river_nml
					generate_rivers:succeeded => !trigger_archive_wrf & !write_empty_river_nml
					generate_rivers:succeeded | write_empty_river_nml => mv_river_files => write_river_number

53
					write_run_namelist => write_river_number => run_fvcom
54 55 56
{% elif RIVER_MODEL == 'NEURAL_NET' %}
					softlink_forcing => generate_rivers => mv_river_files
					mv_river_files & write_run_namelist => write_river_number => run_fvcom
57 58 59 60
{% elif RIVER_MODEL == 'CLIMATOLOGY' %}
					start_cycle => get_river_files => write_run_namelist => write_river_number => run_fvcom
{% endif %}

61
{% if SEDIMENT %}
62
					write_cstms_namelist => run_fvcom
63
{% endif %}
64

65
{% if FORECAST %} 
66
					write_run_namelist => run_fvcom => transfer_data => transfer_data_today
67
{% else %}
68
					write_run_namelist => run_fvcom => transfer_data
69
{% endif %}
70 71
					run_fvcom => nan_check 
					transfer_data => plot_surf_vars
72 73


74 75 76 77 78 79 80 81 82 83 84 85 86
					"""
		[[[+P1D/P1D]]]
			graph = """
					run_fvcom[-P1D] => transfer_restart => run_fvcom
					"""

[runtime]
	[[root]]
		env-script = eval $(rose task-env --cycle-offset=P1D)
		script = rose task-run --verbose
		[[[job]]]
			execution time limit = PT3H
		[[[events]]]
87 88
			mail events = submission timeout, execution timeout, failed
			mail to = {{MAIL_TO}}
89 90
			submission timeout = P1D
		[[[environment]]]
91 92 93 94 95 96 97 98
			START_DATE=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			END_DATE=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			START_DATE_STR=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d_%H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			END_DATE_STR=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d_%H:%M:%S' $CYLC_TASK_CYCLE_POINT)
			START_DAY=$(rose date --offset=-P{{HINDCAST_DAYS|int}}D --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)
			END_DAY=$(rose date --offset=P{{FORECAST_DAYS|int}}D --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)

			FORECAST_DAY=$(rose date --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)
99 100 101 102

			GRID_NAME={{GRID_NAME}}
			RIVER_MODEL_PATH={{RIVER_MODEL_PATH}}
			REMOTE_TRANSFER_DIR={{REMOTE_TRANSFER_DIR}}
103
			ARCHIVE_DIR={{ARCHIVE_DIR}}
104
			PLOT_DIR={{PLOT_DIR}}
105 106

			HARMONICS_FILE_PATH={{HARMONICS_FILE_PATH}}
107
			DONOR_RESTART_FILE_PATH={{DONOR_RESTART_FILE_PATH}}
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122
			CMEMS_DATA_DIR={{CMEMS_DATA_DIR}}
			NEST_RUN_SUITE={{NEST_RUN_SUITE}}
			NEST_INTERVAL={{NEST_INTERVAL}}

			EXT_TSTEP={{EXT_TSTEP}}
			ISPLIT={{ISPLIT}}
			STARTUP_DMAX={{STARTUP_DMAX}}
			HORIZONTAL_MIXING_COEFFICIENT={{HORIZONTAL_MIXING_COEFFICIENT}}
			VERTICAL_MIXING_COEFFICIENT={{VERTICAL_MIXING_COEFFICIENT}}
			COORDINATES={{COORDINATES}}
			WIND_ON={{WIND_ON}}
			HEATING_ON={{HEATING_ON}}
			PRECIPITATION_ON={{PRECIPITATION_ON}}
			AIR_PRESSURE_ON={{AIR_PRESSURE_ON}}

123 124 125
			TEMP_ACTIVE={{TEMP_ACTIVE}}
			SALT_ACTIVE={{SALT_ACTIVE}}

126 127 128 129 130 131 132 133
			NC_INTERVAL={{NC_INTERVAL}}
			NC_AV_ON={{NC_AV_ON}}
			NC_AV_INTERVAL={{NC_AV_INTERVAL}}

			VELOCITY_OUT={{VELOCITY_OUT}}
			SALT_TEMP_OUT={{SALT_TEMP_OUT}}
			TURBULENCE_OUT={{TURBULENCE_OUT}}
			WIND_OUT={{WIND_OUT}}
134
			HEAT_RAIN_OUT={{HEAT_RAIN_OUT}}
135

136 137
			WRF_FORECAST_FILE_DIR={{WRF_FORECAST_FILE_DIR}}
			WRF_ARCHIVE_DIR={{WRF_ARCHIVE_DIR}}
138
			COMMON_FILES_PATH={{COMMON_FILES_PATH}}
139 140 141 142 143 144 145 146 147
                        WRF_ARCHIVE_DATESTR=$(rose date --print-format={{WRF_ARCHIVE_DATEFMT}} $CYLC_TASK_CYCLE_POINT)

{% if FORECAST %}
                        TODAY_WRF_FILE=/{{WRF_ARCHIVE_DIR}}/today/today_wrfout.nc
{% else %}
			TODAY_WRF_FILE='hindcast'
{% endif %}


148
{% if BOUNDARY_FORCING == 'TIDAL' %}
149
			ELEV_FORCE='T'
150
{% else %}
151
			ELEV_FORCE='F'
152
{% endif %}
153

154

155
{% if BOUNDARY_FORCING in ['CMEMS', 'NEST_FILE'] %}
156 157
                        NEST_ON='T'
			NEST_TYPE='3'
158
{% else %}
159 160
                        NEST_ON='T'
			NEST_TYPE='3'
161
{% endif %}
162 163


164 165 166 167 168 169
	[[slurm_job]]
		[[[job]]]
			batch system = slurm
			submission polling intervals = PT10S
			execution polling intervals = PT10S, PT1M
		[[[directives]]]
Mike Bedington's avatar
Mike Bedington committed
170
			--nodes = {{MIN_NODES}}-{{MAX_NODES}}
171 172 173
			--ntasks-per-node=20
			--threads-per-core=1
			--time=24:00:00
174
{% if USE_CETO %}
175 176
		[[[remote]]]
			host = login.ceto.npm.ac.uk
177
			owner = {{REMOTE_USER}}
178
{% endif %}
179 180 181 182 183 184 185 186
	[[slurm_job_1]]
		inherit = slurm_job
		[[[directives]]]
			--nodes = 1
			--ntasks-per-node = 1
			--threads-per-core=1
			--time=24:00:00
	[[remote_job]]
187
{% if USE_CETO %}
188 189
		[[[remote]]]
			host = login.ceto.npm.ac.uk
190
			owner = {{REMOTE_USER}}
191
{% endif %}
192 193 194 195


	[[write_run_namelist]]
		inherit = slurm_job_1
196
{% if SEDIMENT %}
197
	[[write_run_cstms]]
198 199 200 201 202 203
{% endif %}

	[[check_output_dir_len]]
                script = """
                        echo "Need to check suite name doesn't take the fvcom output dir over the max length"
                        """
204 205 206

	[[copy_common_to_remote]]
		inherit = remote_job
207 208
	[[softlink_forcing]]
	[[softlink_forcing_remote]]
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
        inherit = slurm_job_1
	
	[[wrf_suite_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
		[[[job]]]
			execution retry delays = 3*PT15M
	[[get_wrf_file]]
		inherit = slurm_job_1


	[[generate_tide_forcing]]
		inherit = slurm_job_1
224
    	[[CMEMS_download_trigger]]
225 226 227 228 229 230
        script =""
        [[[suite state polling]]]
            interval = PT1M
            max-polls = 1440
        [[[job]]]
            execution retry delays = 3*PT15M
231
	[[generate_CMEMS_nest]]
Mike Bedington's avatar
Mike Bedington committed
232 233 234 235
	[[copy_CMEMS_nest_to_remote]]
		inherit = remote_job
		script = "cp /pml${REMOTE_TRANSFER_DIR}/boundary_nest.nc ${ROSE_DATAC}/${GRID_NAME}_nest_0001.nc"

236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
	[[nest_suite_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
	[[get_nest_file]]
		inherit = slurm_job_1

	[[wrf_archive_trigger]]
		script =""
		[[[suite state polling]]]
			interval = PT1M
			max-polls = 1440
		[[[job]]]
			execution retry delays = 3*PT15M
251 252 253 254 255 256
	
	[[get_archive_wrf_file]]
		inherit = remote_job
	[[convert_wrf_file]]
		inherit = slurm_job_1

257 258 259 260 261 262 263 264 265 266 267 268
	[[update_river_model]]
	[[generate_rivers]]
	[[get_river_files]]
		inherit = slurm_job_1
	[[mv_river_files]]
		inherit = remote_job
	[[write_river_number]]
		inherit = slurm_job_1
		script = """
				NO_RIVERS=$(grep "&NML_RIVER" ${ROSE_DATAC}/${GRID_NAME}_riv.nml | wc -l);
				sed -i "s|.*RIVER_NUMBER.*| RIVER_NUMBER  = ${NO_RIVERS},|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
				"""
269 270 271 272 273 274 275 276 277
	[[trigger_archive_wrf]]
                script = """
                        echo "Get the dates missing from the river model and pass to the wrf archive run" 
                        """

	[[write_empty_river_nml]]
		script = """
			echo "Make a fake riv.nml and copy to run directory" 
			"""
278 279 280

	[[run_fvcom]]
		inherit = slurm_job
281
	[[adjust_namelist]]
282 283 284 285
		inherit = remote_job
		script = """
			sed -i "s|.*STARTUP_UV_TYPE.*|STARTUP_UV_TYPE   = 'default',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
			sed -i "s|.*STARTUP_TURB_TYPE.*|STARTUP_TURB_TYPE = 'default',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
286
			sed -i "s|.*STARTUP_TS_TYPE.*|STARTUP_TS_TYPE   = 'set values',|" ${ROSE_DATAC}/${GRID_NAME}_run.nml
287
{% if SEDIMENT %}
288
			sed -i "s|.*SED_HOT_START.*|SED_HOT_START = F|" ${ROSE_DATAC}/cstms_sediment.inp
289
{% endif %}
290
			"""
291 292
	[[generate_CMEMS_start]]
	[[mv_start_file]]
293
		inherit = remote_job
294 295
	[[transfer_data]]
		inherit = slurm_job_1
296 297
	[[transfer_data_today]]
		inherit = slurm_job_1
298 299 300 301 302 303 304 305 306
	[[start_cycle]]
		script = """
			echo "starting forecast cycle"
			"""
	[[transfer_restart]]
		inherit = slurm_job
		script = """
			ln -s ${ROSE_DATACP1D}/output/${GRID_NAME}_restart_0001.nc ${ROSE_DATAC}/${GRID_NAME}_restart_0001.nc
			"""
307 308
	[[nan_check]]
		inherit = remote_job
309
	[[plot_surf_vars]]