From commits-return-984-apmail-climate-commits-archive=climate.apache.org@climate.apache.org Thu Apr 3 02:25:18 2014 Return-Path: X-Original-To: apmail-climate-commits-archive@minotaur.apache.org Delivered-To: apmail-climate-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A7A06107DD for ; Thu, 3 Apr 2014 02:25:18 +0000 (UTC) Received: (qmail 56421 invoked by uid 500); 3 Apr 2014 02:25:17 -0000 Delivered-To: apmail-climate-commits-archive@climate.apache.org Received: (qmail 56370 invoked by uid 500); 3 Apr 2014 02:25:17 -0000 Mailing-List: contact commits-help@climate.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@climate.apache.org Delivered-To: mailing list commits@climate.apache.org Received: (qmail 56361 invoked by uid 99); 3 Apr 2014 02:25:16 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 03 Apr 2014 02:25:16 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 7CFD1939195; Thu, 3 Apr 2014 02:25:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: joyce@apache.org To: commits@climate.apache.org Date: Thu, 03 Apr 2014 02:25:16 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] git commit: CLIMATE-386 - Add functionality to output a dataset as NetCDF Repository: climate Updated Branches: refs/heads/master 7bde9e126 -> 82396cf4e CLIMATE-386 - Add functionality to output a dataset as NetCDF - Add dataset_processor.write_netcdf for outputting a Dataset object as a NetCDF4 file. - Add unit tests for write_netcdf functionality. Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/97269f40 Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/97269f40 Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/97269f40 Branch: refs/heads/master Commit: 97269f40ebf30c4413a0300a0ec743e507fd2737 Parents: c9558e0 Author: Michael Joyce Authored: Sat Mar 29 16:31:02 2014 -0700 Committer: Michael Joyce Committed: Sat Mar 29 16:31:02 2014 -0700 ---------------------------------------------------------------------- ocw/dataset_processor.py | 48 ++++++++++++++++++++++++++++++++ ocw/tests/test_dataset_processor.py | 25 +++++++++++++++++ 2 files changed, 73 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/97269f40/ocw/dataset_processor.py ---------------------------------------------------------------------- diff --git a/ocw/dataset_processor.py b/ocw/dataset_processor.py index e9adba2..72c0172 100644 --- a/ocw/dataset_processor.py +++ b/ocw/dataset_processor.py @@ -23,6 +23,7 @@ import numpy.ma as ma import scipy.interpolate import scipy.ndimage from scipy.ndimage import map_coordinates +import netCDF4 import logging @@ -210,6 +211,53 @@ def normalize_dataset_datetimes(dataset, timestep): dataset.name ) +def write_netcdf(dataset, path, compress=True): + ''' Write a dataset to a NetCDF file. + + :param dataset: The dataset to write. + :type dataset: ocw.dataset.Dataset + + :param path: The output file path. + :type path: string + ''' + out_file = netCDF4.Dataset(path, 'w', format='NETCDF4') + + # Set attribute lenghts + lat_len = len(dataset.lats) + lon_len = len(dataset.lons) + time_len = len(dataset.times) + + # Create attribute dimensions + lat_dim = out_file.createDimension('lat', lat_len) + lon_dim = out_file.createDimension('lon', lon_len) + time_dim = out_file.createDimension('time', time_len) + + # Create variables + lats = out_file.createVariable('lat', 'f8', ('lat',), zlib=compress) + lons = out_file.createVariable('lon', 'f8', ('lon',), zlib=compress) + times = out_file.createVariable('time', 'f8', ('time',), zlib=compress) + + var_name = dataset.variable if dataset.variable else 'var' + values = out_file.createVariable(var_name, + 'f8', + ('time', 'lat', 'lon'), + zlib=compress) + + # Set the time variable units + # We don't deal with hourly/minutely/anything-less-than-a-day data so + # we can safely stick with a 'days since' offset here. Note that the + # NetCDF4 helper date2num doesn't support 'months' or 'years' instead + # of days. + times.units = "days since %s" % dataset.times[0] + + # Store the dataset's values + lats[:] = dataset.lats + lons[:] = dataset.lons + times[:] = netCDF4.date2num(dataset.times, times.units) + values[:] = dataset.values + + out_file.close() + def _rcmes_normalize_datetimes(datetimes, timestep): """ Normalize Dataset datetime values. http://git-wip-us.apache.org/repos/asf/climate/blob/97269f40/ocw/tests/test_dataset_processor.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_dataset_processor.py b/ocw/tests/test_dataset_processor.py index a144d4a..f883ad9 100644 --- a/ocw/tests/test_dataset_processor.py +++ b/ocw/tests/test_dataset_processor.py @@ -17,8 +17,11 @@ import unittest import datetime +import os + from ocw import dataset_processor as dp from ocw import dataset as ds +from ocw.data_source import local import numpy as np import numpy.ma as ma @@ -268,6 +271,28 @@ class TestFailingSubset(unittest.TestCase): with self.assertRaises(ValueError): dp.subset(self.subregion, self.target_dataset) +class TestNetCDFWrite(unittest.TestCase): + def setUp(self): + self.ds = ten_year_monthly_dataset() + self.file_name = 'test.nc' + + def tearDown(self): + if os.path.isfile(self.file_name): + os.remove(self.file_name) + + def test_file_write(self): + dp.write_netcdf(self.ds, self.file_name) + self.assertTrue(os.path.isfile(self.file_name)) + + def test_that_file_contents_are_valid(self): + dp.write_netcdf(self.ds, self.file_name) + new_ds = local.load_file(self.file_name, self.ds.variable) + + self.assertEqual(self.ds.variable, new_ds.variable) + self.assertTrue(np.array_equal(self.ds.lats, new_ds.lats)) + self.assertTrue(np.array_equal(self.ds.lons, new_ds.lons)) + self.assertTrue(np.array_equal(self.ds.times, new_ds.times)) + self.assertTrue(np.array_equal(self.ds.values, new_ds.values)) def ten_year_monthly_dataset(): lats = np.array(range(-89, 90, 2))