Skip to content

Commit 696cf33

Browse files
authored
Merge pull request #40 from CDAT/fix_percentile_bug
Fix percentile bug
2 parents 8efedd2 + 9af4592 commit 696cf33

File tree

6 files changed

+278
-174
lines changed

6 files changed

+278
-174
lines changed

.circleci/config.yml

Lines changed: 7 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,5 @@
11
version: 2.1
22

3-
parameters:
4-
pkg_name:
5-
type: string
6-
default: "genutil"
7-
repo_name:
8-
type: string
9-
default: "genutil"
10-
last_stable:
11-
type: string
12-
default: "8.2"
13-
user:
14-
type: string
15-
default: "cdat"
16-
label:
17-
type: string
18-
default: "nightly"
19-
env_name:
20-
type: string
21-
default: "test_genutil"
22-
233
aliases:
244
- &setup_env
255
name: setup_env
@@ -31,28 +11,24 @@ aliases:
3111
export PROJECT_DIR=/home/circleci/project/workdir/linux
3212
fi
3313
echo "export WORKDIR=$PROJECT_DIR/$PY_VER" >> $BASH_ENV
34-
cat $BASH_ENV
3514
source $BASH_ENV
3615
mkdir -p $WORKDIR
3716
3817
- &setup_miniconda
3918
name: setup_miniconda
4019
command: |
4120
source $BASH_ENV
42-
mkdir -p $WORKDIR
4321
git clone https://github.com/CDAT/cdat.git $WORKDIR/cdat
4422
# install_miniconda.py installs miniconda3 under $WORKDIR/miniconda
4523
python $WORKDIR/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3'
4624
47-
4825
- &conda_rerender
4926
name: conda_rerender
5027
command: |
5128
source $BASH_ENV
5229
source $WORKDIR/miniconda/etc/profile.d/conda.sh
5330
conda activate base
54-
echo "make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH"
55-
make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH
31+
make conda-rerender workdir=$WORKDIR branch=$CIRCLE_BRANCH
5632
5733
- &conda_build
5834
name: conda_build
@@ -62,7 +38,7 @@ aliases:
6238
conda activate base
6339
os=`uname`
6440
artifacts_dir="artifacts/artifacts.${os}.py_${PY_VER}"
65-
make conda-build conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR artifact_dir=$PWD/$artifacts_dir build_version=$PY_VER
41+
make conda-build workdir=$WORKDIR artifact_dir=$artifacts_dir build_version=$PY_VER
6642
6743
- &setup_run_tests
6844
name: setup_run_tests
@@ -72,19 +48,15 @@ aliases:
7248
conda activate base
7349
export CONDA_PY_VER="python=$PY_VER"
7450
export LIBNETCDF_VER="libnetcdf=*=${LIBNETCDF}_*"
75-
76-
echo "make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs=\"$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS\""
77-
make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs="$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS"
78-
79-
make conda-dump-env conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME artifact_dir=$PWD/spec_artifacts conda_env_filename=$CIRCLE_JOB
51+
make setup-tests workdir=$WORKDIR extra_pkgs="$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS"
52+
make conda-dump-env workdir=$WORKDIR artifact_dir=$PWD/spec_artifacts conda_env_filename=$CIRCLE_JOB
8053
8154
- &run_tests
8255
name: run_tests
8356
command: |
8457
source $BASH_ENV
8558
source $WORKDIR/miniconda/etc/profile.d/conda.sh
86-
conda activate $ENV_NAME
87-
make run-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME workdir=$WORKDIR
59+
make run-tests workdir=$WORKDIR
8860
no_output_timeout: 10m
8961

9062
- &conda_upload
@@ -93,8 +65,8 @@ aliases:
9365
source $BASH_ENV
9466
source $WORKDIR/miniconda/etc/profile.d/conda.sh
9567
conda activate base
96-
UPLOAD_OPTIONS="conda_upload_token=$CONDA_UPLOAD_TOKEN user=$USER label=$LABEL"
97-
make conda-upload $UPLOAD_OPTIONS conda=$WORKDIR/miniconda/bin/conda artifact_dir="$PWD/artifacts/*/"
68+
UPLOAD_OPTIONS="conda_upload_token=$CONDA_UPLOAD_TOKEN"
69+
make conda-upload workdir=$WORKDIR $UPLOAD_OPTIONS artifact_dir="artifacts/*/"
9870
9971
executors:
10072
linux:
@@ -113,9 +85,6 @@ jobs:
11385
type: string
11486
executor: << parameters.os >>
11587
environment:
116-
PKG_NAME: << pipeline.parameters.pkg_name >>
117-
REPO_NAME: << pipeline.parameters.repo_name >>
118-
LAST_STABLE: << pipeline.parameters.last_stable >>
11988
PY_VER: << parameters.py_ver >>
12089
steps:
12190
- checkout
@@ -141,8 +110,6 @@ jobs:
141110
type: string
142111
executor: << parameters.os >>
143112
environment:
144-
PKG_NAME: << pipeline.parameters.pkg_name >>
145-
ENV_NAME: << pipeline.parameters.env_name >>
146113
PY_VER: << parameters.py_ver >>
147114
LIBNETCDF: << parameters.libnetcdf >>
148115
steps:
@@ -164,8 +131,6 @@ jobs:
164131
machine:
165132
image: circleci/classic:latest
166133
environment:
167-
USER: << pipeline.parameters.user >>
168-
LABEL: << pipeline.parameters.label >>
169134
PY_VER: "3.7"
170135
steps:
171136
- checkout

Lib/statistics.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1668,6 +1668,10 @@ def geometricmean(x, axis=0, max_pct_missing=100.):
16681668

16691669

16701670
def _percentiles(out, percent):
1671+
# change 1d to 2d array
1672+
if out.ndim == 1:
1673+
out = out.reshape((1,)+out.shape)
1674+
16711675
if cdms2.isVariable(out):
16721676
out = MV2.sort(out, axis=0).asma()
16731677
ns = MV2.count(out, axis=0).asma()
@@ -1696,10 +1700,8 @@ def _percentiles(out, percent):
16961700
pass
16971701
Aii = numpy.where(numpy.equal(ns, 1), 100., tmp)
16981702
ii = numpy.where(numpy.equal(ii, ns), ns - 1, ii)
1699-
if numpy.rank(ii) > 0:
1703+
if numpy.ndim(ii) > 0:
17001704
ii = ii.astype(numpy.int)
1701-
# tmp = (p-Ai)/(Aii-Ai)*array_indexing.extract(out,ii) + \
1702-
# (Aii-p)/(Aii-Ai)*array_indexing.extract(out,i)
17031705

17041706
tmp = (p - Ai) / (Aii - Ai) * arrayindexing.get(out, ii) + \
17051707
(Aii - p) / (Aii - Ai) * arrayindexing.get(out, i)

Lib/stats_checker.py

Lines changed: 44 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,15 @@
44
from .averager import __check_weightoptions
55

66

7-
class StatisticsError (Exception):
7+
class StatisticsError(Exception):
88
def __init__(self, args=None):
99
"""Create an exception"""
1010
self.args = args
1111

1212
def __str__(self):
1313
"""Calculate the string representation"""
1414
return str(self.args)
15+
1516
__repr__ = __str__
1617

1718

@@ -32,10 +33,10 @@ def __makeweights(x, w, axes):
3233
if not numpy.ma.isarray(w):
3334
# Ok Krishna returned a list of 1D arrays.... Let's put it together
3435
axs = x.getAxisList()
35-
axes = cdms2.order2index(axs, axes)[:len(cdms2.orderparse(axes))]
36+
axes = cdms2.order2index(axs, axes)[: len(cdms2.orderparse(axes))]
3637
endax = []
3738
for i in range(len(axes)):
38-
if w[i] == 'unweighted':
39+
if w[i] == "unweighted":
3940
w[i] = numpy.ma.ones(len(axs[axes[i]]), dtype=x.dtype.char)
4041
if i == 0:
4142
wo = w[i]
@@ -45,8 +46,8 @@ def __makeweights(x, w, axes):
4546
endax.append(axs[axes[i]])
4647
w = cdms2.MV2.array(wo)
4748
w.setAxisList(endax)
48-
# else:
49-
# w.setAxisList(x.getAxisList())
49+
# else:
50+
# w.setAxisList(x.getAxisList())
5051
return w
5152

5253

@@ -64,29 +65,31 @@ def __checker(x, y, w, axes, smally=0):
6465
x = numpy.ma.array(x, copy=0)
6566
if not numpy.ma.isarray(y) and y is not None:
6667
y = numpy.ma.array(y, copy=0)
67-
if not numpy.ma.isarray(
68-
w) and w is not None and not isinstance(w, type('')):
69-
if not isinstance(w[0], type('')):
68+
if not numpy.ma.isarray(w) and w is not None and not isinstance(w, type("")):
69+
if not isinstance(w[0], type("")):
7070
w = numpy.ma.array(w, copy=0)
7171
else:
7272
if not xismv:
7373
raise StatisticsError(
74-
'Error if weights are a list then x must be an MV2 !!!')
74+
"Error if weights are a list then x must be an MV2 !!!"
75+
)
7576
w = __makeweights(x, w, axes)
7677
wismv = 1
7778
elif w is not None:
7879
if not xismv:
7980
raise StatisticsError(
80-
'Error if weights are a list then x must be an MV2 !!!')
81+
"Error if weights are a list then x must be an MV2 !!!"
82+
)
8183
w = __makeweights(x, w, axes)
8284
wismv = 1
8385

8486
if xismv * yismv * wismv != 1:
8587
# We didn't pass all MV2s shapes have to match (unless None)
8688
if smally == 0:
8789
if x.shape != numpy.ma.shape(y) and y is not None:
88-
raise StatisticsError('Error x and y shape do not match !' +
89-
str(x.shape) + ',' + str(numpy.ma.shape(y)))
90+
raise StatisticsError(
91+
"Error x and y shape do not match !" + str(x.shape) + "," + str(numpy.ma.shape(y))
92+
)
9093
else:
9194
shy = list(y.shape)
9295
shy2 = y.shape
@@ -96,7 +99,9 @@ def __checker(x, y, w, axes, smally=0):
9699
for i in axes:
97100
myaxes.append(eval(i))
98101
elif isinstance(axes, int):
99-
myaxes = [axes, ]
102+
myaxes = [
103+
axes,
104+
]
100105
else:
101106
myaxes = list(axes)
102107
for anaxis in myaxes[::-1]:
@@ -109,25 +114,27 @@ def __checker(x, y, w, axes, smally=0):
109114
sh[i] = myaxes[i]
110115
y = numpy.ma.transpose(y, sh)
111116
if x.shape != numpy.ma.shape(y) and y is not None:
112-
raise StatisticsError('Error x and y shape do not match (y shouldbe 1D less than x) !' +
113-
str(x.shape) + ',' + str(shy2) + ' Remember y must be 1D less than x')
117+
err_msg = "Error x and y shape do not match (y shouldbe 1D less than x) !"
118+
raise StatisticsError(
119+
err_msg + str(x.shape) + "," + str(shy2) + " Remember y must be 1D less than x"
120+
)
114121
if x.shape != numpy.ma.shape(w) and w is not None:
115-
raise StatisticsError('Error x and weights shape do not match !' +
116-
str(x.shape) + ',' + str(numpy.ma.shape(w)) +
117-
' ATTENTION if you are trynig to pass a list of 1D arrays' +
118-
'for each dim, then x must be an MV2 !!!')
122+
msg1 = "Error x and weights shape do not match !"
123+
msg2 = " ATTENTION if you are trying to pass a list of 1D arrays for each dim, then x must be an MV2 !!!"
124+
raise StatisticsError(msg1 + str(x.shape) + "," + str(numpy.ma.shape(w)) + msg2)
119125
if not isinstance(axes, type([])):
120126
axes = cdms2.orderparse(str(axes))
121127
for i in axes:
122128
if len(x.shape) < i:
123-
raise StatisticsError('Error you have ' + str(len(x.shape)) +
124-
' dimensions and try to work on dim:' + str(i))
129+
err_msg = "Error you have " + str(len(x.shape)) + " dimensions and try to work on dim:" + str(i)
130+
raise StatisticsError(err_msg)
125131
else:
126132
if y is not None:
127133
x, y = grower(x, y)
128134
if x.shape != y.shape:
129135
raise StatisticsError(
130-
'Error x and y have different shapes' + str(x.shape) + ', ' + str(y.shape))
136+
"Error x and y have different shapes" + str(x.shape) + ", " + str(y.shape)
137+
)
131138
ax = x.getAxisList()
132139
xorder = x.getOrder(ids=1)
133140
# Now grows w
@@ -136,11 +143,13 @@ def __checker(x, y, w, axes, smally=0):
136143
for o in worder:
137144
if o not in xorder:
138145
raise StatisticsError(
139-
'Error weights have a dimension that is neither in x or y:' + o)
146+
"Error weights have a dimension that is neither in x or y:" + o
147+
)
140148
x, w = grower(x, w)
141149
if x.shape != w.shape:
142150
raise StatisticsError(
143-
'Error x and weights have different shapes' + str(x.shape) + ', ' + str(w.shape))
151+
"Error x and weights have different shapes" + str(x.shape) + ", " + str(w.shape)
152+
)
144153
# Last thing convert the axes input to numbers
145154
if isinstance(axes, type(1)):
146155
axes = str(axes)
@@ -149,24 +158,21 @@ def __checker(x, y, w, axes, smally=0):
149158
naxes = len(axesparse)
150159
for i in range(naxes):
151160
o = axesparse[i]
152-
if isinstance(o, type('')):
161+
if isinstance(o, type("")):
153162
for j in range(len(xorder)):
154163
if xorder[j] == o:
155164
axesparse[i] = j
156165
# Well it must be a name for x y t....
157-
if isinstance(axesparse[i], type('')):
166+
if isinstance(axesparse[i], type("")):
158167
for j in range(len(x.shape)):
159168
if o[1:-1] == x.getAxis(j).id:
160169
axesparse[i] = j
161170
# Everything failed the axis id must be not existing in the
162171
# slab...
163-
if isinstance(axesparse[i], type('')):
172+
if isinstance(axesparse[i], type("")):
164173
raise StatisticsError(
165-
'Error axis id :' +
166-
o +
167-
' not found in first slab: ' +
168-
x.getOrder(
169-
ids=1))
174+
"Error axis id :" + o + " not found in first slab: " + x.getOrder(ids=1)
175+
)
170176
axes = axesparse
171177
# Now we have array those shape match, and a nice list of axes let's keep
172178
# going
@@ -176,8 +182,12 @@ def __checker(x, y, w, axes, smally=0):
176182
xorder = list(range(len(x.shape)))
177183
forder = []
178184
for i in range(naxes):
179-
forder.append(axes[i])
180-
n0 = n0 * xsh[axes[i]]
185+
a = axes[i]
186+
forder.append(a)
187+
try:
188+
n0 = n0 * xsh[a]
189+
except IndexError:
190+
raise Exception("Axis {} is out of bounds for dimension {}".format(a, len(xsh)))
181191
fsh = [n0]
182192
ax2 = []
183193
for i in range(len(x.shape)):

0 commit comments

Comments
 (0)