Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
R
rose_regrid_setup
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Mike Bedington
rose_regrid_setup
Commits
0dca4407
Commit
0dca4407
authored
May 10, 2019
by
Modellers Operational
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Convert to running the interpolation on CETO
parent
34cb5a60
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
53 additions
and
26 deletions
+53
-26
app/regrid_domain_ceto/file/add_FillValue.sh
app/regrid_domain_ceto/file/add_FillValue.sh
+7
-0
app/regrid_domain_ceto/file/grid_interp.py
app/regrid_domain_ceto/file/grid_interp.py
+24
-19
app/regrid_domain_ceto/rose-app.conf
app/regrid_domain_ceto/rose-app.conf
+4
-0
app/softlink_to_portal/rose-app.conf
app/softlink_to_portal/rose-app.conf
+6
-0
app/transfer_from_remote/rose-app.conf
app/transfer_from_remote/rose-app.conf
+1
-2
rose-suite.conf
rose-suite.conf
+5
-3
suite.rc
suite.rc
+6
-2
No files found.
app/regrid_domain_ceto/file/add_FillValue.sh
0 → 100644
View file @
0dca4407
#!/bin/sh
vars_str
=
$1
vars_list
=
$(
echo
$vars_str
|
tr
","
"
\n
"
)
for
v
in
$vars_list
;
do
ncatted
-a
_FillValue,
$v
,o,f,
"-32768"
output.nc
done
app/regrid_domain_ceto/file/grid_interp.py
View file @
0dca4407
...
...
@@ -65,45 +65,50 @@ if rank == 0:
collected_interp_data
[
this_var
]
=
np
.
ma
.
masked_invalid
(
np
.
vstack
(
output_list
))
# write to cmems format
output_file
=
'output.nc'
fvcom
=
pf
.
read
.
FileReader
(
fvcom_file
)
dims
=
{
'time'
:
len
(
fvcom
.
time
.
datetime
),
'depth'
:
len
(
worker
.
regular_grid
.
dep_lays
),
'lon'
:
len
(
worker
.
regular_grid
.
lons
),
'lat'
:
len
(
worker
.
regular_grid
.
lats
),
'DateStrLen'
:
26
}
all_attributes
=
{
'lon'
:{
'standard_name'
:
'longitude'
,
'units'
:
'degrees_east'
},
'lat'
:{
'standard_name'
:
'latitude'
,
'units'
:
'degrees_north'
},
'depth'
:{
'standard_name'
:
'depth'
,
'units'
:
'm'
},
'temp'
:{
'standard_name'
:
'sea_water_potential_temperature'
,
'units'
:
'C'
,
'missing_value'
:
-
32768
}
,
'salinity'
:{
'standard_name'
:
'sea_water_salinity'
,
'units'
:
'psu'
,
'missing_value'
:
-
32768
},
'u'
:{
'standard_name'
:
'eastward_sea_water_velocity'
,
'units'
:
'm s-1'
,
'missing_value'
:
32768
},
'v'
:{
'standard_name'
:
'northward_sea_water_velocity'
,
'units'
:
'm s-1'
,
'missing_value'
:
-
32768
},
'zeta'
:{
'standard_name'
:
'sea_surface_height_above_geoid'
,
'units'
:
'm'
,
'missing_value'
:
-
32768
}}
output_file_name
=
'output.nc'
all_attributes
=
{
'lon'
:{
'standard_name'
:
'longitude'
,
'units'
:
'degrees_east'
,
'long_name'
:
'longitude'
},
'lat'
:{
'standard_name'
:
'latitude'
,
'units'
:
'degrees_north'
,
'long_name'
:
'latitude'
},
'depth'
:{
'standard_name'
:
'depth'
,
'units'
:
'm'
,
'long_name'
:
'depth, measured downwards from free surface'
,
'axis'
:
'Z'
,
'positive'
:
'down'
},
'temp'
:{
'standard_name'
:
'sea_water_potential_temperature'
,
'units'
:
'C'
,
'missing_value'
:
-
32768
,
'long_name'
:
'Sea Water Potential Temperature'
}
,
'salinity'
:{
'standard_name'
:
'sea_water_salinity'
,
'units'
:
'psu'
,
'missing_value'
:
-
32768
,
'long_name'
:
'Sea Water Salinity'
},
'u'
:{
'standard_name'
:
'eastward_sea_water_velocity'
,
'units'
:
'm s-1'
,
'missing_value'
:
-
32768
,
'long_name'
:
'Eastward Current Velocity'
},
'v'
:{
'standard_name'
:
'northward_sea_water_velocity'
,
'units'
:
'm s-1'
,
'missing_value'
:
-
32768
,
'long_name'
:
'Northward Current Velocity'
},
'zeta'
:{
'standard_name'
:
'sea_surface_height_above_geoid'
,
'units'
:
'm'
,
'missing_value'
:
-
32768
,
'long_name'
:
'Sea surface height above geoid'
}}
globals
=
{
'type'
:
'OPERATIONAL MODEL REGULAR GRID OUTPUT'
,
'title'
:
'Regularly gridded data interpolated from FVCOM output'
,
'history'
:
'File created using PyFVCOM'
,
'filename'
:
str
(
output_file
),
'Conventions'
:
'CF-1.0'
}
'title'
:
'Regularly gridded data interpolated from FVCOM output'
,
'history'
:
'File created using PyFVCOM'
,
'filename'
:
str
(
output_file_name
),
'Conventions'
:
'CF-1.0'
,
'_FillValue'
:
-
32768
}
for
this_val
in
collected_interp_data
.
values
():
np
.
ma
.
set_fill_value
(
this_val
,
-
32768
)
ncopts
=
{}
with
pf
.
preproc
.
WriteForcing
(
output_file
,
dims
,
global_attributes
=
globals
,
clobber
=
True
,
format
=
'NETCDF4'
)
as
outfile
:
#for this_var, this_mode in varlist.items():
# this_output_file_name = 'output_{}.nc'.format(this_var)
with
pf
.
preproc
.
WriteForcing
(
output_file_name
,
dims
,
global_attributes
=
globals
,
clobber
=
True
,
format
=
'NETCDF4'
)
as
outfile
:
# Add the variables.
outfile
.
write_fvcom_time
(
fvcom
.
time
.
datetime
)
outfile
.
add_variable
(
'lon'
,
worker
.
regular_grid
.
lons
,
[
'lon'
],
attributes
=
all_attributes
[
'lon'
])
outfile
.
add_variable
(
'lat'
,
worker
.
regular_grid
.
lats
,
[
'lat'
],
attributes
=
all_attributes
[
'lat'
])
outfile
.
add_variable
(
'depth'
,
worker
.
regular_grid
.
dep_lays
,
[
'depth'
],
attributes
=
all_attributes
[
'depth'
])
for
this_var
,
this_mode
in
varlist
.
items
():
if
this_mode
==
'surface'
:
outfile
.
add_variable
(
this_var
,
collected_interp_data
[
this_var
],
[
'time'
,
'lon'
,
'lat'
],
attributes
=
all_attributes
[
this_var
],
ncopts
=
ncopts
)
attributes
=
all_attributes
[
this_var
],
ncopts
=
ncopts
)
else
:
outfile
.
add_variable
(
this_var
,
collected_interp_data
[
this_var
],
[
'time'
,
'lon'
,
'lat'
,
'depth'
],
attributes
=
all_attributes
[
this_var
],
ncopts
=
ncopts
)
outfile
.
write_fvcom_time
(
fvcom
.
time
.
datetime
)
attributes
=
all_attributes
[
this_var
],
ncopts
=
ncopts
)
else
:
print
(
'Rank {} process finished'
.
format
(
rank
))
app/regrid_domain_ceto/rose-app.conf
View file @
0dca4407
...
...
@@ -15,4 +15,8 @@ default =
export
I_MPI_PMI_LIBRARY
=/
usr
/
lib64
/
libpmi
.
so
;
srun
-
n
$
np
python3
grid_interp
.
py
${
FVCOM_GRID_NAME
}
_
0001
.
nc
${
GRID_LOWER_LEFT
} ${
GRID_UPPER_RIGHT
} ${
HORIZ_RES
} ${
DEPTH_LAYERS
} ${
VARS
};
module
purge
;
module
load
nco
;
bash
./
add_FillValue
.
sh
${
VARS
};
mv
output
.
nc
${
ROSE_DATAC
}
app/softlink_to_portal/rose-app.conf
0 → 100644
View file @
0dca4407
[
command
]
default
=
src_file
=/${
OUTPUT_DIR
}/${
FORECAST_DAY
}/${
OUTPUT_FILENAME
}.
nc
;
mkdir
-
p
/${
PORTAL_BASEDIR
}/${
FORECAST_YEAR
}/${
FORECAST_MONTH
}/
dst_file
=/${
PORTAL_BASEDIR
}/${
FORECAST_YEAR
}/${
FORECAST_MONTH
}/${
OUTPUT_FILENAME
}
_
hrly_
${
FORECAST_DAY
}.
nc
ln
-
sf
${
src_file
} ${
dst_file
}
app/transfer_from_remote/rose-app.conf
View file @
0dca4407
...
...
@@ -2,5 +2,4 @@
default
=
dst
=/
pml
${
OUTPUT_DIR
}/${
FORECAST_DAY
};
mkdir
-
p
$
dst
;
src
=${
ROSE_DATAC
}/
output
.
nc
;
ssh
ceto6
-
t
"rsync -aph --no-o --no-g $src $dst/${OUTPUT_FILENAME}"
;
ssh
ceto6
-
t
"rsync -aph --no-o --no-g $src $dst/${OUTPUT_FILENAME}.nc"
;
rose-suite.conf
View file @
0dca4407
[
jinja2
:
suite
.
rc
]
## Run properties
INITIAL_START_DATE
=
'2019-0
3-18
T00:00:00Z'
INITIAL_START_DATE
=
'2019-0
5-04
T00:00:00Z'
FINAL_CYCLE_POINT
=
'NONE'
MAIL_TO
=
'mbe@pml.ac.uk'
NPROCS
=
12
...
...
@@ -16,8 +16,10 @@ FVCOM_OUTPUT_DIR='data/sthenno1/scratch/modop/Model/FVCOM_tamar/output'
GRID_LOWER_LEFT
=
'-4.3,50.24'
GRID_UPPER_RIGHT
=
'-4.05,50.55'
HORIZ_RES
=
0
.
001
DEPTH_LAYERS
=
'0,
2,5,10,15,25
,40'
DEPTH_LAYERS
=
'0,
3,5,10,15,20,25,30
,40'
VARS
=
'temp,salinity,u,v,zeta'
OUTPUT_DIR
=
'data/sthenno1/scratch/modop/Model/FVCOM_tamar/estuary_output'
OUTPUT_FILENAME
=
'tamar_estuary_0001.nc'
OUTPUT_FILENAME
=
'tamar_estuary_001'
PORTAL_BASEDIR
=
'data/sthenno1/scratch/modop/Portal/Model/TAMAR_ESTUARY_FORECAST_PHY_001/tamar_estuary_forecast_phy_001_hourly_t_s_u_v_ssh'
suite.rc
View file @
0dca4407
...
...
@@ -15,7 +15,7 @@
graph = """
{% if USE_CETO %}
regrid_domain_ceto[-P1D]:finish => start_cycle => suite_trigger <{{TRIGGER_SUITE}}::{{TRIGGER_TASK}}> => transfer_to_remote
transfer_to_remote => regrid_domain_ceto => transfer_from_remote
transfer_to_remote => regrid_domain_ceto => transfer_from_remote
=> softlink_to_portal
{% else %}
regrid_domain[-P1D]:finish => start_cycle => suite_trigger <{{TRIGGER_SUITE}}::{{TRIGGER_TASK}}> => regrid_domain
{% endif %}
...
...
@@ -43,8 +43,11 @@
OUTPUT_DIR={{OUTPUT_DIR}}
OUTPUT_FILENAME={{OUTPUT_FILENAME}}
FORECAST_DAY=$(rose date --print-format='%Y-%m-%d' $CYLC_TASK_CYCLE_POINT)
FORECAST_YEAR=$(rose date --print-format='%Y' $CYLC_TASK_CYCLE_POINT)
FORECAST_MONTH=$(rose date --print-format='%m' $CYLC_TASK_CYCLE_POINT)
FVCOM_FILE={{FVCOM_OUTPUT_DIR}}/${FORECAST_DAY}/${FVCOM_GRID_NAME}_0001.nc
PORTAL_BASEDIR={{PORTAL_BASEDIR}}
[[start_cycle]]
script = """
...
...
@@ -86,6 +89,7 @@
inherit = remote_job
[[regrid_domain_ceto]]
inherit = slurm_job
[[softlink_to_portal]]
{% else %}
[[regrid_domain]]
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment