Add GFS processing script

app/gfs_proc_data/file/proc_gfs_file.sh

+#!/bin/bash
+
+# Make symlinks for gfs files and purge old ones
+
+set -eu
+
+cd "$1" # switch to gfs forecast directory
+
+dirlist=(gfs*/)
+
+if [ $# -gt 1 ]; then
+	remove_days=$2 # Delete forecast folders from longer than this number of days ago
+	# Remove folders of forecasts older than cutoff
+	cut_date=$(date +%s --date="${remove_days} days ago")
+	for this_dir in "${dirlist[@]}"; do
+		this_YmdH=$(echo ${this_dir} | cut -d'_' -f 2)
+		this_Ymd=${this_YmdH::${#this_YmdH}-2}
+		this_H=${this_YmdH:8}
+
+		dir_date=$(date +%s -d "${this_Ymd} ${this_H}00")
+
+		if [ $dir_date -lt $cut_date ]; then
+			rm -r $this_dir
+		fi
+	done
+fi
+
+# Symlink forcing to timestamped files in forecast_data folder
+# It relies on dirlist automatically sorting by name then reversing for using the latest possible forecasts for each file
+cd forecast_data
+rm gfs_forecast_* || true
+
+for ((i=${#dirlist[@]}-1; i>=0; i--)); do
+	this_dir=${dirlist[i]}
+
+	this_YmdH=$(echo ${this_dir} | cut -d'_' -f 2)
+	this_Ymd=${this_YmdH::${#this_YmdH}-2}
+	this_H=${this_YmdH:8}
+
+	for file in ../${this_dir}*; do
+		[ -e "$file" ] || continue
+		f_hr_raw=$(echo ${file} | cut -d'.' -f 7)
+		f_hr=${f_hr_raw:1}
+
+		filename=gfs_forecast_$(date +%Y%m%d_%H%M -d "${this_Ymd} ${this_H}00 +$f_hr hour")
+
+		if [ ! -f ${filename} ]; then
+			ln -s ../${this_dir}/${file} ${filename}
+		fi
+	done
+done
+
+

app/gfs_proc_data/rose-app.conf

+[command]
+default = bash ./proc_gfs_file.sh ${CETO_GFS_DIR} ${CETO_DATE_CUTOFF};

rose-suite.conf

 [jinja2:suite.rc]
 ## Run properties
-INITIAL_START_DATE='2018-11-27T12:00:00Z'
+INITIAL_START_DATE='2019-06-19T12:00:00Z'
 FINAL_CYCLE_POINT='NONE'
 MAIL_TO='mbe@pml.ac.uk'
+USE_CETO=True
+REMOTE_USER='modop'
 
 # GFS settings
+# GFS_FTP_DIR='ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/'
+GFS_FTP_DIR='ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/'
+
 EURYALE_GFS_ARCHIVE='/data/sthenno1/scratch/modop/Data/GFS/'
 CETO_GFS_DIR='/gpfs1/users/modellers/modop/Data/GFS_rolling/'
+CETO_DATE_CUTOFF=40
 
 ## CMEMS settings
 CMEMS_DATA_DIR='/data/sthenno1/scratch/modop/Data/CMEMS'

suite.rc

@@ -13,7 +13,7 @@
 	[[dependencies]]
 		[[[PT6H]]]
 			graph = """
-					finish_download[-PT6H] => start_cycle => get_GFS_data:finish  & get_CMEMS_data:finish => finish_download
+					finish_download[-PT6H] => start_cycle => get_GFS_data:finish  & get_CMEMS_data:finish => gfs_proc_data:finish => finish_download
 					"""
 
 [runtime]
@@ -29,10 +29,18 @@
 		[[[environment]]]
 			START_DATE=$(rose date --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
 			END_DATE=$(rose date --offset=P1D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
-
 			CMEMS_DATA_DIR={{CMEMS_DATA_DIR}}
 			EURYALE_GFS_ARCHIVE={{EURYALE_GFS_ARCHIVE}}
 			CETO_GFS_DIR={{CETO_GFS_DIR}}			
+			CETO_DATE_CUTOFF={{CETO_DATE_CUTOFF}}
+			GFS_FTP_DIR={{GFS_FTP_DIR}}
+        [[remote_job]]
+{%- if USE_CETO %}
+                [[[remote]]]
+                        host = login.ceto.npm.ac.uk
+                        owner = {{REMOTE_USER}}
+{%- endif %}
+
 
 	[[start_cycle]]
 		script = """
@@ -40,6 +48,8 @@
 				"""
 	[[get_CMEMS_data]]
 	[[get_GFS_data]]
+	[[gfs_proc_data]]
+                inherit = remote_job		
 	[[finish_download]]
         script = """
             echo "finished download for "${START_DATE}