Commit e9767212 authored by Modellers Operational's avatar Modellers Operational

Add GFS processing script

parent c860bc99
#!/bin/bash
# Make symlinks for gfs files and purge old ones
set -eu
cd "$1" # switch to gfs forecast directory
dirlist=(gfs*/)
if [ $# -gt 1 ]; then
remove_days=$2 # Delete forecast folders from longer than this number of days ago
# Remove folders of forecasts older than cutoff
cut_date=$(date +%s --date="${remove_days} days ago")
for this_dir in "${dirlist[@]}"; do
this_YmdH=$(echo ${this_dir} | cut -d'_' -f 2)
this_Ymd=${this_YmdH::${#this_YmdH}-2}
this_H=${this_YmdH:8}
dir_date=$(date +%s -d "${this_Ymd} ${this_H}00")
if [ $dir_date -lt $cut_date ]; then
rm -r $this_dir
fi
done
fi
# Symlink forcing to timestamped files in forecast_data folder
# It relies on dirlist automatically sorting by name then reversing for using the latest possible forecasts for each file
cd forecast_data
rm gfs_forecast_* || true
for ((i=${#dirlist[@]}-1; i>=0; i--)); do
this_dir=${dirlist[i]}
this_YmdH=$(echo ${this_dir} | cut -d'_' -f 2)
this_Ymd=${this_YmdH::${#this_YmdH}-2}
this_H=${this_YmdH:8}
for file in ../${this_dir}*; do
[ -e "$file" ] || continue
f_hr_raw=$(echo ${file} | cut -d'.' -f 7)
f_hr=${f_hr_raw:1}
filename=gfs_forecast_$(date +%Y%m%d_%H%M -d "${this_Ymd} ${this_H}00 +$f_hr hour")
if [ ! -f ${filename} ]; then
ln -s ../${this_dir}/${file} ${filename}
fi
done
done
[command]
default = bash ./proc_gfs_file.sh ${CETO_GFS_DIR} ${CETO_DATE_CUTOFF};
[jinja2:suite.rc] [jinja2:suite.rc]
## Run properties ## Run properties
INITIAL_START_DATE='2018-11-27T12:00:00Z' INITIAL_START_DATE='2019-06-19T12:00:00Z'
FINAL_CYCLE_POINT='NONE' FINAL_CYCLE_POINT='NONE'
MAIL_TO='mbe@pml.ac.uk' MAIL_TO='mbe@pml.ac.uk'
USE_CETO=True
REMOTE_USER='modop'
# GFS settings # GFS settings
# GFS_FTP_DIR='ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/'
GFS_FTP_DIR='ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/'
EURYALE_GFS_ARCHIVE='/data/sthenno1/scratch/modop/Data/GFS/' EURYALE_GFS_ARCHIVE='/data/sthenno1/scratch/modop/Data/GFS/'
CETO_GFS_DIR='/gpfs1/users/modellers/modop/Data/GFS_rolling/' CETO_GFS_DIR='/gpfs1/users/modellers/modop/Data/GFS_rolling/'
CETO_DATE_CUTOFF=40
## CMEMS settings ## CMEMS settings
CMEMS_DATA_DIR='/data/sthenno1/scratch/modop/Data/CMEMS' CMEMS_DATA_DIR='/data/sthenno1/scratch/modop/Data/CMEMS'
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
[[dependencies]] [[dependencies]]
[[[PT6H]]] [[[PT6H]]]
graph = """ graph = """
finish_download[-PT6H] => start_cycle => get_GFS_data:finish & get_CMEMS_data:finish => finish_download finish_download[-PT6H] => start_cycle => get_GFS_data:finish & get_CMEMS_data:finish => gfs_proc_data:finish => finish_download
""" """
[runtime] [runtime]
...@@ -29,10 +29,18 @@ ...@@ -29,10 +29,18 @@
[[[environment]]] [[[environment]]]
START_DATE=$(rose date --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT) START_DATE=$(rose date --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
END_DATE=$(rose date --offset=P1D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT) END_DATE=$(rose date --offset=P1D --print-format='%Y-%m-%d %H:%M:%S' $CYLC_TASK_CYCLE_POINT)
CMEMS_DATA_DIR={{CMEMS_DATA_DIR}} CMEMS_DATA_DIR={{CMEMS_DATA_DIR}}
EURYALE_GFS_ARCHIVE={{EURYALE_GFS_ARCHIVE}} EURYALE_GFS_ARCHIVE={{EURYALE_GFS_ARCHIVE}}
CETO_GFS_DIR={{CETO_GFS_DIR}} CETO_GFS_DIR={{CETO_GFS_DIR}}
CETO_DATE_CUTOFF={{CETO_DATE_CUTOFF}}
GFS_FTP_DIR={{GFS_FTP_DIR}}
[[remote_job]]
{%- if USE_CETO %}
[[[remote]]]
host = login.ceto.npm.ac.uk
owner = {{REMOTE_USER}}
{%- endif %}
[[start_cycle]] [[start_cycle]]
script = """ script = """
...@@ -40,6 +48,8 @@ ...@@ -40,6 +48,8 @@
""" """
[[get_CMEMS_data]] [[get_CMEMS_data]]
[[get_GFS_data]] [[get_GFS_data]]
[[gfs_proc_data]]
inherit = remote_job
[[finish_download]] [[finish_download]]
script = """ script = """
echo "finished download for "${START_DATE} echo "finished download for "${START_DATE}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment