Correcting ISF and SF

parent 04eec379
......@@ -64,6 +64,10 @@ Estimation methods
``````````````````
Here are the methods implemented in pyqt_fit. To access these methods, the simplest is to use the instances provided.
.. py:data:: unbounded
Instance of the :py:class:`KDE1DMethod` class.
.. py:data:: renormalization
Instance of the :py:class:`RenormalizationMethod` class.
......
......@@ -413,7 +413,7 @@ class KDE1D(object):
"""
self.fit_if_needed()
return self.method.sf(self, np.asleast1d(points), out)
return self.method.sf(self, np.atleast_1d(points), out)
def sf_grid(self, N=None, cut=None):
r"""
......
This diff is collapsed.
......@@ -4,6 +4,7 @@ from ..utils import namedtuple
from ..compat import irange
from scipy import stats, special
from .. import kernels
from .. import kde
def generate(dist, N, low, high):
start = dist.cdf(low)
......@@ -13,7 +14,7 @@ def generate(dist, N, low, high):
def setupClass_norm(cls):
cls.dist = stats.norm(0, 1)
cls.sizes = np.array([512, 1024, 2048])
cls.sizes = [512, 1024, 2048]
cls.vs = [generate(cls.dist, s, -5, 5) for s in cls.sizes]
cls.args = {}
cls.weights = [ cls.dist.pdf(v) for v in cls.vs ]
......@@ -21,10 +22,11 @@ def setupClass_norm(cls):
cls.xs = np.r_[-5:5:1024j]
cls.lower = -5
cls.upper = 5
cls.methods = methods
def setupClass_lognorm(cls):
cls.dist = stats.lognorm(1)
cls.sizes = np.array([512, 1024, 2048])
cls.sizes = [512, 1024, 2048]
cls.args = {}
cls.vs = [ generate(cls.dist, s, 0.001, 20) for s in cls.sizes ]
cls.vs = [ v[v < 20] for v in cls.vs ]
......@@ -33,10 +35,11 @@ def setupClass_lognorm(cls):
cls.lambdas = [ 1 - ws for ws in cls.weights ]
cls.lower = 0
cls.upper = 20
cls.methods = methods_log
test_method = namedtuple('test_method', ['instance', 'accuracy', 'grid_accuracy', 'bound_low', 'bound_high'])
methods = [ test_method(None, 1e-5, 1e-4, False, False)
methods = [ test_method(kde_methods.unbounded, 1e-5, 1e-4, False, False)
, test_method(kde_methods.reflection, 1e-5, 1e-4, True, True)
, test_method(kde_methods.cyclic, 1e-5, 1e-4, True, True)
, test_method(kde_methods.renormalization, 1e-5, 1e-4, True, True)
......@@ -55,42 +58,71 @@ kernels1d = [ test_kernel(kernels.normal_kernel1d, 1, 1, True)
class KDETester(object):
def createKDE(self, data, method, **args):
all_args = dict(self.args)
all_args.update(args)
k = kde.KDE1D(data, **all_args)
if method.instance is None:
del k.method
else:
k.method = method.instance
if method.bound_low:
k.lower = self.lower
else:
del k.lower
if method.bound_high:
k.upper = self.upper
else:
del k.upper
assert k.fitted is False
return k
def test_methods(self):
for m in methods:
for i in irange(len(self.sizes)):
yield self.method_works, i, m
k = self.createKDE(self.vs[i], m)
yield self.method_works, k, m, '{0}_{1}'.format(m.instance, i)
def test_grid_methods(self):
for m in methods:
for m in self.methods:
for i in irange(len(self.sizes)):
yield self.grid_method_works, i, m
k = self.createKDE(self.vs[i], m)
yield self.grid_method_works, k, m, '{0}_{1}'.format(m.instance, i)
def test_weights_methods(self):
for m in methods:
for m in self.methods:
for i in irange(len(self.sizes)):
yield self.weights_method_works, i, m
k = self.createKDE(self.vs[i], m)
k.weights = self.weights[i]
yield self.method_works, k, m, 'weights_{0}_{1}'.format(m.instance, i)
def test_weights_grid_methods(self):
for m in methods:
for m in self.methods:
for i in irange(len(self.sizes)):
yield self.weights_grid_method_works, i, m
k = self.createKDE(self.vs[i], m)
k.weights=self.weights[i]
yield self.grid_method_works, k, m, 'weights_{0}_{1}'.format(m.instance, i)
def test_lambdas_methods(self):
for m in methods:
for m in self.methods:
for i in irange(len(self.sizes)):
yield self.lambdas_method_works, i, m
k = self.createKDE(self.vs[i], m)
k.lambdas = self.lambdas[i]
yield self.method_works, k, m, 'lambdas_{0}_{1}'.format(m.instance, i)
def test_lambdas_grid_methods(self):
for m in methods:
for m in self.methods:
for i in irange(len(self.sizes)):
yield self.lambdas_grid_method_works, i, m
k = self.createKDE(self.vs[i], m)
k.lambdas = self.lambdas[i]
yield self.grid_method_works, k, m, 'lambdas_{0}_{1}'.format(m.instance, i)
def kernel_works_(self, k):
self.kernel_works(k)
self.kernel_works(k, 'default')
if kernels.HAS_CYTHON:
try:
kernels.usePython()
self.kernel_works(k)
self.kernel_works(k, 'python')
finally:
kernels.useCython()
......@@ -99,11 +131,11 @@ class KDETester(object):
yield self.kernel_works_, k
def grid_kernel_works_(self, k):
self.grid_kernel_works(k)
self.grid_kernel_works(k, 'default')
if kernels.HAS_CYTHON:
try:
kernels.usePython()
self.grid_kernel_works(k)
self.grid_kernel_works(k, 'python')
finally:
kernels.useCython()
......
......@@ -16,27 +16,7 @@ class TestCDF(kde_utils.KDETester):
def setUpClass(cls):
kde_utils.setupClass_lognorm(cls)
def createKDE(self, data, method, **args):
all_args = dict(self.args)
all_args.update(args)
k = kde.KDE1D(data, **all_args)
if method.instance is None:
del k.method
else:
k.method = method.instance
if method.bound_low:
k.lower = self.lower
else:
del k.lower
if method.bound_high:
k.upper = self.upper
else:
del k.upper
assert k.fitted is False
return k
def method_works(self, i, method, **args):
k = self.createKDE(self.vs[i], method, **args)
def method_works(self, k, method, name):
begin, last = k.cdf([k.lower, k.upper])
assert abs(last - 1) < method.accuracy, "Error, k.cdf({0}) = {1} should be close to 1".format(k.upper, last)
assert abs(begin) < method.accuracy, "Error, k.cdf({0}) = {1} should be close to 0".format(k.lower, begin)
......@@ -55,6 +35,7 @@ class TestCDF(kde_utils.KDETester):
def numeric_cdf(self, i, method):
k = self.createKDE(self.vs[i], method)
k.fit()
ys = k.method.numeric_cdf(k, self.xs)
xxs, yys = k.method.numeric_cdf_grid(k, N=2**12)
ys2 = np.interp(self.xs, xxs, yys)
......@@ -69,40 +50,14 @@ class TestCDF(kde_utils.KDETester):
def test_numeric_cdf(self):
self.numeric_cdf(0, kde_utils.methods[0])
def weights_method_works(self, i, method):
return self.method_works(i, method, weights = self.weights[i])
def lambdas_method_works(self, i, method):
return self.method_works(i, method, lambdas = self.lambdas[i])
def grid_method_works(self, i, method):
k = self.createKDE(self.vs[i], method)
xs, ys = k.cdf_grid(64)
acc = method.accuracy
assert np.all(ys >= -acc), "Some negative values"
assert np.all(ys <= 1+acc), "CDF must be below one"
assert np.all(ys[1:] - ys[:-1] >= -acc), "The CDF must be strictly growing."
def weights_grid_method_works(self, i, method):
weights = self.weights[i]
k = self.createKDE(self.vs[i], method)
k.weights = self.weights[i]
xs, ys = k.cdf_grid(64)
acc = method.accuracy
assert np.all(ys >= -acc), "Some negative values"
assert np.all(ys <= 1+acc), "CDF must be below one"
assert np.all(ys[1:] - ys[:-1] >= -acc), "The CDF must be strictly growing."
def lambdas_grid_method_works(self, i, method):
k = self.createKDE(self.vs[i], method)
k.lambdas = self.lambdas[i]
def grid_method_works(self, k, method, name):
xs, ys = k.cdf_grid(64)
acc = method.accuracy
assert np.all(ys >= -acc), "Some negative values"
assert np.all(ys <= 1+acc), "CDF must be below one"
assert np.all(ys[1:] - ys[:-1] >= -acc), "The CDF must be strictly growing."
def kernel_works(self, ker):
def kernel_works(self, ker, name):
method = kde_utils.methods[0]
k = self.createKDE(self.vs[1], method)
k.kernel = ker.cls()
......@@ -111,7 +66,7 @@ class TestCDF(kde_utils.KDETester):
assert abs(last - 1) < acc, "Error, k.cdf({0}) = {1} should be close to 1".format(k.upper, last)
assert abs(begin) < acc, "Error, k.cdf({0}) = {1} should be close to 0".format(k.lower, begin)
def grid_kernel_works(self, ker):
def grid_kernel_works(self, ker, name):
method = kde_utils.methods[0]
k = self.createKDE(self.vs[1], method)
k.kernel = ker.cls()
......
......@@ -47,75 +47,26 @@ class TestKDE1D(kde_utils.KDETester):
@classmethod
def setUpClass(cls):
kde_utils.setupClass_norm(cls)
cls.methods = kde_utils.methods
def createKDE(self, data, method, **args):
all_args = dict(self.args)
all_args.update(args)
k = kde.KDE1D(data, **all_args)
if method.instance is None:
del k.method
else:
k.method = method.instance
if method.bound_low:
k.lower = self.lower
else:
del k.lower
if method.bound_high:
k.upper = self.upper
else:
del k.upper
assert k.fitted is False
return k
#def test_converge(self):
#xs = np.r_[-3:3:512j]
#ys = self.dist.pdf(xs)
#ks = [ self.createKDE(v, **self.args) for v in self.vs ]
def method_works(self, i, method):
k = self.createKDE(self.vs[i], method, **self.args)
def method_works(self, k, method, name):
k.fit()
tot = integrate.quad(k.pdf, k.lower, k.upper, limit=100)[0]
assert abs(tot - 1) < method.accuracy, "Error, {} should be close to 1".format(tot)
def grid_method_works(self, i, method):
k = self.createKDE(self.vs[i], method, **self.args)
xs, ys = k.grid(4000)
tot = integrate.simps(ys, xs)
assert abs(tot - 1) < method.grid_accuracy, "Error, {} should be close to 1".format(tot)
def weights_method_works(self, i, method):
weights = self.weights[i]
k = self.createKDE(self.vs[i], method, weights=weights, **self.args)
tot = integrate.quad(k.pdf, k.lower, k.upper, limit=100)[0]
assert abs(tot - 1) < method.accuracy, "Error, {} should be close to 1".format(tot)
del k.weights
k.fit()
assert k.total_weights == len(self.vs[i])
def weights_grid_method_works(self, i, method):
weights = self.weights[i]
k = self.createKDE(self.vs[i], method, weights=weights, **self.args)
xs, ys = k.grid(2048)
tot = integrate.simps(ys, xs)
assert abs(tot - 1) < method.grid_accuracy, "Error, {} should be close to 1".format(tot)
def lambdas_method_works(self, i, method):
lambdas = self.lambdas[i]
k = self.createKDE(self.vs[i], method, lambdas=lambdas, **self.args)
tot = integrate.quad(k.pdf, k.lower, k.upper, limit=100)[0]
assert abs(tot - 1) < method.accuracy, "Error, {} should be close to 1".format(tot)
del k.lambdas
k.fit()
assert k.lambdas == 1
assert k.total_weights == len(k.xdata)
assert k.lambdas == 1.
def lambdas_grid_method_works(self, i, method):
lambdas = self.lambdas[i]
k = self.createKDE(self.vs[i], method, lambdas=lambdas, **self.args)
xs, ys = k.grid(2048)
def grid_method_works(self, k, method, name):
xs, ys = k.grid(4000)
tot = integrate.simps(ys, xs)
assert abs(tot - 1) < method.accuracy, "Error, {} should be close to 1".format(tot)
assert abs(tot - 1) < method.grid_accuracy, "Error, {} should be close to 1".format(tot)
def test_copy(self):
k = self.createKDE(self.vs[0], self.methods[0])
......@@ -143,7 +94,7 @@ class TestKDE1D(kde_utils.KDETester):
assert k.fitted is not None
k.fit()
def kernel_works(self, ker):
def kernel_works(self, ker, name):
method = self.methods[0]
k = self.createKDE(self.vs[1], method)
k.kernel = ker.cls()
......@@ -151,7 +102,7 @@ class TestKDE1D(kde_utils.KDETester):
acc = method.grid_accuracy * ker.precision_factor
assert abs(tot - 1) < acc, "Error, {} should be close to 1".format(tot)
def grid_kernel_works(self, ker):
def grid_kernel_works(self, ker, name):
method = self.methods[0]
k = self.createKDE(self.vs[1], method)
xs, ys = k.grid()
......@@ -159,9 +110,71 @@ class TestKDE1D(kde_utils.KDETester):
acc = method.grid_accuracy * ker.precision_factor
assert abs(tot - 1) < acc, "Error, {} should be close to 1".format(tot)
class LogTestKDE1D(kde_utils.KDETester):
class LogTestKDE1D(TestKDE1D):
@classmethod
def setUpClass(cls):
kde_utils.setupClass_lognorm(cls)
class TestSF(kde_utils.KDETester):
@classmethod
def setUpClass(cls):
kde_utils.setupClass_norm(cls)
cls.methods = kde_utils.methods
del cls.sizes[1:]
def method_works(self, k, method, name):
k.fit()
xs = kde_methods.generate_grid(k)
sf = k.sf(xs)
cdf = k.cdf(xs)
np.testing.assert_allclose(sf, 1-cdf, method.accuracy, method.accuracy)
def grid_method_works(self, k, method, name):
xs, sf = k.sf_grid()
_, cdf = k.cdf_grid()
np.testing.assert_allclose(sf, 1-cdf, method.accuracy, method.accuracy)
def kernel_works(self, ker, name):
pass
def grid_kernel_works(self, ker, name):
pass
class TestLogSF(TestSF):
@classmethod
def setUpClass(cls):
kde_utils.setupClass_lognorm(cls)
del cls.sizes[1:]
class TestISF(kde_utils.KDETester):
@classmethod
def setUpClass(cls):
kde_utils.setupClass_norm(cls)
cls.methods = kde_utils.methods
del cls.sizes[1:]
def method_works(self, k, method, name):
k.fit()
sf = np.linspace(0, 1, 64)
sf_xs = k.isf(sf)
cdf_xs = k.icdf(1-sf)
np.testing.assert_allclose(sf_xs, cdf_xs, method.accuracy, method.accuracy)
def grid_method_works(self, k, method, name):
comp_sf, xs = k.isf_grid()
ref_sf = k.sf(xs)
np.testing.assert_allclose(comp_sf, ref_sf, 0.03, 0.03)
def kernel_works(self, ker, name):
pass
def grid_kernel_works(self, ker, name):
pass
class TestLogISF(TestISF):
@classmethod
def setUpClass(cls):
kde_utils.setupClass_lognorm(cls)
cls.methods = kde_utils.methods_log
del cls.sizes[1:]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment