Here's a small function that calculates decriptive statistics for frequency distributions:
# from __future__ import division (for Python 2)
def descriptives_from_agg(values, freqs):
values = np.array(values)
freqs = np.array(freqs)
arg_sorted = np.argsort(values)
values = values[arg_sorted]
freqs = freqs[arg_sorted]
count = freqs.sum()
fx = values * freqs
mean = fx.sum() / count
variance = ((freqs * values**2).sum() / count) - mean**2
variance = count / (count - 1) * variance # dof correction for sample variance
std = np.sqrt(variance)
minimum = np.min(values)
maximum = np.max(values)
cumcount = np.cumsum(freqs)
Q1 = values[np.searchsorted(cumcount, 0.25*count)]
Q2 = values[np.searchsorted(cumcount, 0.50*count)]
Q3 = values[np.searchsorted(cumcount, 0.75*count)]
idx = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max']
result = pd.Series([count, mean, std, minimum, Q1, Q2, Q3, maximum], index=idx)
return result
A demo:
np.random.seed(0)
val = np.random.normal(100, 5, 1000).astype(int)
pd.Series(val).describe()
Out:
count 1000.000000
mean 99.274000
std 4.945845
min 84.000000
25% 96.000000
50% 99.000000
75% 103.000000
max 113.000000
dtype: float64
vc = pd.value_counts(val)
descriptives_from_agg(vc.index, vc.values)
Out:
count 1000.000000
mean 99.274000
std 4.945845
min 84.000000
25% 96.000000
50% 99.000000
75% 103.000000
max 113.000000
dtype: float64
Note that this doesn't handle NaN's and is not properly tested.
与恶龙缠斗过久,自身亦成为恶龙;凝视深渊过久,深渊将回以凝视…