move some stuff to OnlineStatsBase

Author: joshday

REQUIRE

@@ -1,5 +1,5 @@
 julia 0.7
-OnlineStatsBase 0.9 0.10
+OnlineStatsBase 0.10 0.11
 LearnBase
 StatsBase
 DataStructures

src/OnlineStats.jl

@@ -3,8 +3,8 @@ module OnlineStats
 using RecipesBase, Reexport, Statistics, LinearAlgebra, Dates
 @reexport using OnlineStatsBase, LossFunctions, PenaltyFunctions, LearnBase
 
-import OnlineStatsBase: OnlineStat, name, _fit!, _merge!, eachrow, Mean, Variance,
-    smooth, smooth!, smooth_syr!, bessel
+import OnlineStatsBase: OnlineStat, name, _fit!, _merge!, eachrow, smooth, smooth!,
+    smooth_syr!, bessel, StatCollection, Mean, Variance, Series, FTSeries
 import LearnBase: fit!, nobs, value, predict, transform, transform!
 import StatsBase: autocov, autocor, confint, skewness, kurtosis, entropy, midpoints,
     fweights, sample, coef, Histogram

src/stats/stats.jl

@@ -1,18 +1,3 @@
-#-----------------------------------------------------# StatCollection (Series and Group)
-abstract type StatCollection{T} <: OnlineStat{T} end
-
-function Base.show(io::IO, o::StatCollection)
-    print(io, name(o, false, false))
-    print_stat_tree(io, o.stats)
-end
-
-function print_stat_tree(io::IO, stats)
-    for (i, stat) in enumerate(stats)
-        char = i == length(stats) ? '└' : '├'
-        print(io, "\n  $(char)── $stat")
-    end
-end
-
 #-----------------------------------------------------------------------# AutoCov and Lag
 # Lag
 """
@@ -206,12 +191,6 @@ nkeys(o::CountMap) = length(o.value)
 Base.values(o::CountMap) = values(o.value)
 Base.getindex(o::CountMap, i) = o.value[i]
 
-#-----------------------------------------------------------------------# Counter
-mutable struct Counter <: OnlineStat{Any}
-    n::Int
-end
-_fit!(o::Counter) = (o.n += 1)
-
 #-----------------------------------------------------------------------# CovMatrix
 """
     CovMatrix(p=0; weight=EqualWeight())
@@ -307,109 +286,6 @@ end
 Base.last(o::Diff) = o.lastval
 Base.diff(o::Diff) = o.diff
 
-
-#-----------------------------------------------------------------------# Extrema
-"""
-    Extrema(T::Type = Float64)
-
-Maximum and minimum.
-
-# Example
-
-    o = fit!(Extrema(), rand(10^5))
-    extrema(o)
-    maximum(o)
-    minimum(o)
-"""
-# T is type to store data, S is type of single observation.
-# E.g. you may want to accept any Number even if you are storing values as Float64
-mutable struct Extrema{T,S} <: OnlineStat{S}
-    min::T
-    max::T
-    n::Int
-    function Extrema(T::Type = Float64)
-        a, b, S = extrema_init(T)
-        new{T,S}(a, b, 0)
-    end
-end
-extrema_init(T::Type{<:Number}) = typemax(T), typemin(T), Number
-extrema_init(T::Type{String}) = "", "", String
-extrema_init(T::Type{Date}) = typemax(Date), typemin(Date), Date
-extrema_init(T::Type) = rand(T), rand(T), T
-function _fit!(o::Extrema, y)
-    (o.n += 1) == 1 && (o.min = o.max = y)
-    o.min = min(o.min, y)
-    o.max = max(o.max, y)
-end
-function _merge!(o::Extrema, o2::Extrema)
-    o.min = min(o.min, o2.min)
-    o.max = max(o.max, o2.max)
-    o.n += o2.n
-    o
-end
-value(o::Extrema) = (o.min, o.max)
-Base.extrema(o::Extrema) = value(o)
-Base.maximum(o::Extrema) = o.max
-Base.minimum(o::Extrema) = o.min
-
-#-----------------------------------------------------------------------# FTSeries
-"""
-    FTSeries(stats...; filter=x->true, transform=identity)
-
-Track multiple stats for one data stream that is filtered and transformed before being
-fitted.
-
-    FTSeries(T, stats...; filter, transform)
-
-Create an FTSeries and specify the type `T` of the transformed values.
-
-# Example
-
-    o = FTSeries(Mean(), Variance(); transform=abs)
-    fit!(o, -rand(1000))
-
-    # Remove missing values represented as DataValues
-    using DataValues
-    y = DataValueArray(randn(100), rand(Bool, 100))
-    o = FTSeries(DataValue, Mean(); transform=get, filter=!isna)
-    fit!(o, y)
-"""
-mutable struct FTSeries{N, OS<:Tup, F, T} <: StatCollection{Union{N,Missing}}
-    stats::OS
-    filter::F
-    transform::T
-    nfiltered::Int
-end
-function FTSeries(stats::OnlineStat...; filter=x->true, transform=identity)
-    Ts = input.(stats)
-    FTSeries{Union{Ts...}, typeof(stats), typeof(filter), typeof(transform)}(
-        stats, filter, transform, 0
-    )
-end
-function FTSeries(T::Type, stats::OnlineStat...; filter=x->true, transform=identity)
-    FTSeries{T, typeof(stats), typeof(filter), typeof(transform)}(stats, filter, transform, 0)
-end
-value(o::FTSeries) = value.(o.stats)
-nobs(o::FTSeries) = nobs(o.stats[1])
-@generated function _fit!(o::FTSeries{N, OS}, y) where {N, OS}
-    n = length(fieldnames(OS))
-    quote
-        if o.filter(y)
-            yt = o.transform(y)
-            Base.Cartesian.@nexprs $n i -> @inbounds begin
-                _fit!(o.stats[i], yt)
-            end
-        else
-            o.nfiltered += 1
-        end
-    end
-end
-function _merge!(o::FTSeries, o2::FTSeries)
-    o.nfiltered += o2.nfiltered
-    _merge!.(o.stats, o2.stats)
-    o
-end
-
 #-----------------------------------------------------------------------# Group
 """
     Group(stats::OnlineStat...)
@@ -1118,57 +994,6 @@ function _merge!(o::T, o2::T) where {T<:ReservoirSample}
     end
 end
 
-#-----------------------------------------------------------------------# Series
-"""
-    Series(stats)
-    Series(stats...)
-    Series(; stats...)
-
-Track a collection stats for one data stream.
-
-# Example
-
-    s = Series(Mean(), Variance())
-    fit!(s, randn(1000))
-"""
-struct Series{IN, T} <: StatCollection{IN}
-    stats::T
-    function Series(stats::T) where T
-        IN = Union{map(input, stats)...}
-        new{IN, T}(stats)
-    end
-end
-Series(t::OnlineStat...) = Series(t)
-Series(; t...) = Series(t.data)
-
-value(o::Series) = map(value, o.stats)
-nobs(o::Series) = nobs(o.stats[1])
-@generated function _fit!(o::Series{IN, T}, y) where {IN, T}
-    n = length(fieldnames(T))
-    :(Base.Cartesian.@nexprs $n i -> _fit!(o.stats[i], y))
-end
-_merge!(o::Series, o2::Series) = map(_merge!, o.stats, o2.stats)
-
-#-----------------------------------------------------------------------# Sum
-"""
-    Sum(T::Type = Float64)
-
-Track the overall sum.
-
-# Example
-
-    fit!(Sum(Int), fill(1, 100))
-"""
-mutable struct Sum{T} <: OnlineStat{Number}
-    sum::T
-    n::Int
-end
-Sum(T::Type = Float64) = Sum(T(0), 0)
-Base.sum(o::Sum) = o.sum
-_fit!(o::Sum{T}, x::Real) where {T<:AbstractFloat} = (o.sum += convert(T, x); o.n += 1)
-_fit!(o::Sum{T}, x::Real) where {T<:Integer} =       (o.sum += round(T, x); o.n += 1)
-_merge!(o::T, o2::T) where {T <: Sum} = (o.sum += o2.sum; o.n += o2.n; o)
-
 # #-----------------------------------------------------------------------# Summarizer
 # mutable struct Summarizer{T} <: OnlineStat{T}
 #     group::Group

test/runtests.jl

@@ -101,26 +101,6 @@ end
     @test diff(o) == 1
     @test last(o) == 10
 end
-#-----------------------------------------------------------------------# Extrema
-@testset "Extrema" begin
-    o = fit!(Extrema(), y)
-    @test extrema(o) == extrema(y)
-    @test minimum(o) == minimum(y)
-    @test maximum(o) == maximum(y)
-
-    @test value(fit!(Extrema(Bool), x)) == extrema(x)
-    @test value(fit!(Extrema(Int), z)) == extrema(z)
-
-    @test ==(mergevals(Extrema(), y, y2)...)
-
-    o = fit!(Extrema(Date), Date(2010):Day(1):Date(2011))
-    @test minimum(o) == Date(2010)
-    @test maximum(o) == Date(2011)
-
-    @test value(fit!(Extrema(Char), 'a':'z')) == ('a', 'z')
-    @test value(fit!(Extrema(Char), "abc")) == ('a', 'c')
-    @test value(fit!(Extrema(String), ["a", "b"])) == ("a", "b")
-end
 #-----------------------------------------------------------------------# Fit[Dist]
 @testset "Fit[Dist]" begin
     @testset "FitBeta" begin
@@ -226,16 +206,6 @@ end
     @test classify(o, randn(10)) in 1:2
     @test mean(classify(o, X) .== Y) > .5
 end
-#-----------------------------------------------------------------------# FTSeries
-@testset "FTSeries" begin
-    o = fit!(FTSeries(Mean(); transform=abs), y)
-    @test value(o)[1] ≈ mean(abs, y)
-
-    data = vcat(y, fill(missing, 20))   
-    o = fit!(FTSeries(Mean(); transform=abs, filter=!ismissing), data)
-    @test value(o)[1] ≈ mean(abs, y) 
-    @test o.nfiltered == 20
-end
 #-----------------------------------------------------------------------# Group
 @testset "Group" begin
     o = fit!(5Mean(), OnlineStatsBase.eachrow(ymat))
@@ -266,7 +236,7 @@ end
     end
 end
 #-----------------------------------------------------------------------# HeatMap
-@testset "HeatMap" begin 
+@testset "HeatMap" begin
     data1 = OnlineStatsBase.eachrow(ymat[:, 1:2])
     data2 = OnlineStatsBase.eachrow(ymat2[:, 1:2])
     @test ==(mergevals(HeatMap(-5:.1:5, -5:.1:5), data1, data2)...)
@@ -283,7 +253,7 @@ end
                 @test fit!(Hist(edges, Number; closed=false, left=false), data).counts == w2
             end
         end
-    end 
+    end
     o = fit!(Hist(-5:.1:5), y)
     for (v1, v2) in zip(extrema(o), extrema(y))
         @test ≈(v1, v2; atol=.1)
@@ -370,7 +340,7 @@ end
     ≈(mergevals(LinReg(), OnlineStatsBase.eachrow(ymat, y), OnlineStatsBase.eachrow(ymat2, y2))...)
 
     o = fit!(LinReg(), (ymat, y))
-    @test coef(o) ≈ ymat \ y 
+    @test coef(o) ≈ ymat \ y
     @test coef(o, .1) ≈ (ymat'ymat ./ n + .1I) \ ymat'y ./ n
     @test coef(o, .1:.1:.5) ≈ (ymat'ymat ./ n + Diagonal(.1:.1:.5)) \ ymat'y ./ n
     @test predict(o, ymat) == ymat * o.β
@@ -511,16 +481,6 @@ end
         @test (yi ∈ y) || (yi ∈ y2)
     end
 end
-#-----------------------------------------------------------------------# Series
-@testset "Series" begin
-    a, b = mergevals(Series(Mean(), Variance()), y, y2)
-    @test a[1] ≈ b[1]
-    @test a[2] ≈ b[2]
-
-    a, b = mergevals(Series(m=Mean(), v=Variance()), y, y2)
-    @test a.m ≈ b.m
-    @test a.v ≈ b.v
-end
 #-----------------------------------------------------------------------# StatHistory
 @testset "StatHistory" begin
     o = fit!(StatHistory(Mean(), 10), 1:20)
@@ -560,23 +520,14 @@ end
         println()
     end
 end
-#-----------------------------------------------------------------------# Sum
-@testset "Sum" begin
-    @test value(fit!(Sum(Int), x)) == sum(x)
-    @test value(fit!(Sum(), y)) ≈ sum(y)
-    @test value(fit!(Sum(Int), z)) == sum(z)
-
-    @test ==(mergevals(Sum(Int), x, x2)...)
-    @test ≈(mergevals(Sum(), y, y2)...)
-    @test ==(mergevals(Sum(Int), z, z2)...)
-end
+#-----------------------------------------------------------------------# Kahan
 
 include("test_kahan.jl")
 
 #-----------------------------------------------------------------------# Show methods
 @testset "Show methods" begin
-    for stat in [BiasVec([1,2,3]), Bootstrap(Mean()), CallFun(Mean(), println), FastNode(5), 
-                 FastTree(5), FastForest(5), FTSeries(Variance()), Group(Mean(), Mean()), 
+    for stat in [BiasVec([1,2,3]), Bootstrap(Mean()), CallFun(Mean(), println), FastNode(5),
+                 FastTree(5), FastForest(5), FTSeries(Variance()), Group(Mean(), Mean()),
                  HyperLogLog{10}(), LinRegBuilder(4), NBClassifier(5, Float64), ProbMap(Int),
                  P2Quantile(.5), Series(Mean()), StatLearn(5)]
         println("  > ", stat)