/************************************************************************* ALGLIB 3.17.0 (source code generated 2020-12-27) Copyright (c) Sergey Bochkanov (ALGLIB project). >>> SOURCE LICENSE >>> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation (www.fsf.org); either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. A copy of the GNU General Public License is available at http://www.fsf.org/licensing/licenses >>> END OF LICENSE >>> *************************************************************************/ #pragma warning disable 162 #pragma warning disable 164 #pragma warning disable 219 using System; public partial class alglib { /************************************************************************* Principal components analysis This function builds orthogonal basis where first axis corresponds to direction with maximum variance, second axis maximizes variance in the subspace orthogonal to first axis and so on. This function builds FULL basis, i.e. returns N vectors corresponding to ALL directions, no matter how informative. If you need just a few (say, 10 or 50) of the most important directions, you may find it faster to use one of the reduced versions: * pcatruncatedsubspace() - for subspace iteration based method It should be noted that, unlike LDA, PCA does not use class labels. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: X - dataset, array[0..NPoints-1,0..NVars-1]. matrix contains ONLY INDEPENDENT VARIABLES. NPoints - dataset size, NPoints>=0 NVars - number of independent variables, NVars>=1 OUTPUT PARAMETERS: Info - return code: * -4, if SVD subroutine haven't converged * -1, if wrong parameters has been passed (NPoints<0, NVars<1) * 1, if task is solved S2 - array[0..NVars-1]. variance values corresponding to basis vectors. V - array[0..NVars-1,0..NVars-1] matrix, whose columns store basis vectors. -- ALGLIB -- Copyright 25.08.2008 by Bochkanov Sergey *************************************************************************/ public static void pcabuildbasis(double[,] x, int npoints, int nvars, out int info, out double[] s2, out double[,] v) { info = 0; s2 = new double[0]; v = new double[0,0]; pca.pcabuildbasis(x, npoints, nvars, ref info, ref s2, ref v, null); } public static void pcabuildbasis(double[,] x, int npoints, int nvars, out int info, out double[] s2, out double[,] v, alglib.xparams _params) { info = 0; s2 = new double[0]; v = new double[0,0]; pca.pcabuildbasis(x, npoints, nvars, ref info, ref s2, ref v, _params); } /************************************************************************* Principal components analysis This function performs truncated PCA, i.e. returns just a few most important directions. Internally it uses iterative eigensolver which is very efficient when only a minor fraction of full basis is required. Thus, if you need full basis, it is better to use pcabuildbasis() function. It should be noted that, unlike LDA, PCA does not use class labels. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: X - dataset, array[0..NPoints-1,0..NVars-1]. matrix contains ONLY INDEPENDENT VARIABLES. NPoints - dataset size, NPoints>=0 NVars - number of independent variables, NVars>=1 NNeeded - number of requested components, in [1,NVars] range; this function is efficient only for NNeeded<=0 NVars - number of independent variables, NVars>=1 NNeeded - number of requested components, in [1,NVars] range; this function is efficient only for NNeeded<1) * -1, incorrect pararemets were passed (N<=0). * 1, OK Threshold- partiton boundary. Left part contains values which are strictly less than Threshold. Right part contains values which are greater than or equal to Threshold. PAL, PBL- probabilities P(0|v=Threshold) and P(1|v>=Threshold) CVE - cross-validation estimate of cross-entropy -- ALGLIB -- Copyright 22.05.2008 by Bochkanov Sergey *************************************************************************/ public static void dsoptimalsplit2(double[] a, int[] c, int n, out int info, out double threshold, out double pal, out double pbl, out double par, out double pbr, out double cve) { info = 0; threshold = 0; pal = 0; pbl = 0; par = 0; pbr = 0; cve = 0; bdss.dsoptimalsplit2(a, c, n, ref info, ref threshold, ref pal, ref pbl, ref par, ref pbr, ref cve, null); } public static void dsoptimalsplit2(double[] a, int[] c, int n, out int info, out double threshold, out double pal, out double pbl, out double par, out double pbr, out double cve, alglib.xparams _params) { info = 0; threshold = 0; pal = 0; pbl = 0; par = 0; pbr = 0; cve = 0; bdss.dsoptimalsplit2(a, c, n, ref info, ref threshold, ref pal, ref pbl, ref par, ref pbr, ref cve, _params); } /************************************************************************* Optimal partition, internal subroutine. Fast version. Accepts: A array[0..N-1] array of attributes array[0..N-1] C array[0..N-1] array of class labels TiesBuf array[0..N] temporaries (ties) CntBuf array[0..2*NC-1] temporaries (counts) Alpha centering factor (0<=alpha<=1, recommended value - 0.05) BufR array[0..N-1] temporaries BufI array[0..N-1] temporaries Output: Info error code (">0"=OK, "<0"=bad) RMS training set RMS error CVRMS leave-one-out RMS error Note: content of all arrays is changed by subroutine; it doesn't allocate temporaries. -- ALGLIB -- Copyright 11.12.2008 by Bochkanov Sergey *************************************************************************/ public static void dsoptimalsplit2fast(ref double[] a, ref int[] c, ref int[] tiesbuf, ref int[] cntbuf, ref double[] bufr, ref int[] bufi, int n, int nc, double alpha, out int info, out double threshold, out double rms, out double cvrms) { info = 0; threshold = 0; rms = 0; cvrms = 0; bdss.dsoptimalsplit2fast(ref a, ref c, ref tiesbuf, ref cntbuf, ref bufr, ref bufi, n, nc, alpha, ref info, ref threshold, ref rms, ref cvrms, null); } public static void dsoptimalsplit2fast(ref double[] a, ref int[] c, ref int[] tiesbuf, ref int[] cntbuf, ref double[] bufr, ref int[] bufi, int n, int nc, double alpha, out int info, out double threshold, out double rms, out double cvrms, alglib.xparams _params) { info = 0; threshold = 0; rms = 0; cvrms = 0; bdss.dsoptimalsplit2fast(ref a, ref c, ref tiesbuf, ref cntbuf, ref bufr, ref bufi, n, nc, alpha, ref info, ref threshold, ref rms, ref cvrms, _params); } } public partial class alglib { /************************************************************************* Model's errors: * RelCLSError - fraction of misclassified cases. * AvgCE - acerage cross-entropy * RMSError - root-mean-square error * AvgError - average error * AvgRelError - average relative error NOTE 1: RelCLSError/AvgCE are zero on regression problems. NOTE 2: on classification problems RMSError/AvgError/AvgRelError contain errors in prediction of posterior probabilities *************************************************************************/ public class modelerrors : alglibobject { // // Public declarations // public double relclserror { get { return _innerobj.relclserror; } set { _innerobj.relclserror = value; } } public double avgce { get { return _innerobj.avgce; } set { _innerobj.avgce = value; } } public double rmserror { get { return _innerobj.rmserror; } set { _innerobj.rmserror = value; } } public double avgerror { get { return _innerobj.avgerror; } set { _innerobj.avgerror = value; } } public double avgrelerror { get { return _innerobj.avgrelerror; } set { _innerobj.avgrelerror = value; } } public modelerrors() { _innerobj = new mlpbase.modelerrors(); } public override alglib.alglibobject make_copy() { return new modelerrors((mlpbase.modelerrors)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private mlpbase.modelerrors _innerobj; public mlpbase.modelerrors innerobj { get { return _innerobj; } } public modelerrors(mlpbase.modelerrors obj) { _innerobj = obj; } } /************************************************************************* *************************************************************************/ public class multilayerperceptron : alglibobject { // // Public declarations // public multilayerperceptron() { _innerobj = new mlpbase.multilayerperceptron(); } public override alglib.alglibobject make_copy() { return new multilayerperceptron((mlpbase.multilayerperceptron)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private mlpbase.multilayerperceptron _innerobj; public mlpbase.multilayerperceptron innerobj { get { return _innerobj; } } public multilayerperceptron(mlpbase.multilayerperceptron obj) { _innerobj = obj; } } /************************************************************************* This function serializes data structure to string. Important properties of s_out: * it contains alphanumeric characters, dots, underscores, minus signs * these symbols are grouped into words, which are separated by spaces and Windows-style (CR+LF) newlines * although serializer uses spaces and CR+LF as separators, you can replace any separator character by arbitrary combination of spaces, tabs, Windows or Unix newlines. It allows flexible reformatting of the string in case you want to include it into text or XML file. But you should not insert separators into the middle of the "words" nor you should change case of letters. * s_out can be freely moved between 32-bit and 64-bit systems, little and big endian machines, and so on. You can serialize structure on 32-bit machine and unserialize it on 64-bit one (or vice versa), or serialize it on SPARC and unserialize on x86. You can also serialize it in C# version of ALGLIB and unserialize in C++ one, and vice versa. *************************************************************************/ public static void mlpserialize(multilayerperceptron obj, out string s_out) { alglib.serializer s = new alglib.serializer(); s.alloc_start(); mlpbase.mlpalloc(s, obj.innerobj, null); s.sstart_str(); mlpbase.mlpserialize(s, obj.innerobj, null); s.stop(); s_out = s.get_string(); } /************************************************************************* This function unserializes data structure from string. *************************************************************************/ public static void mlpunserialize(string s_in, out multilayerperceptron obj) { alglib.serializer s = new alglib.serializer(); obj = new multilayerperceptron(); s.ustart_str(s_in); mlpbase.mlpunserialize(s, obj.innerobj, null); s.stop(); } /************************************************************************* This function serializes data structure to stream. Data stream generated by this function is same as string representation generated by string version of serializer - alphanumeric characters, dots, underscores, minus signs, which are grouped into words separated by spaces and CR+LF. We recommend you to read comments on string version of serializer to find out more about serialization of AlGLIB objects. *************************************************************************/ public static void mlpserialize(multilayerperceptron obj, System.IO.Stream stream_out) { alglib.serializer s = new alglib.serializer(); s.alloc_start(); mlpbase.mlpalloc(s, obj.innerobj, null); s.sstart_stream(stream_out); mlpbase.mlpserialize(s, obj.innerobj, null); s.stop(); } /************************************************************************* This function unserializes data structure from stream. *************************************************************************/ public static void mlpunserialize(System.IO.Stream stream_in, out multilayerperceptron obj) { alglib.serializer s = new alglib.serializer(); obj = new multilayerperceptron(); s.ustart_stream(stream_in); mlpbase.mlpunserialize(s, obj.innerobj, null); s.stop(); } /************************************************************************* Creates neural network with NIn inputs, NOut outputs, without hidden layers, with linear output layer. Network weights are filled with small random values. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreate0(int nin, int nout, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreate0(nin, nout, network.innerobj, null); } public static void mlpcreate0(int nin, int nout, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreate0(nin, nout, network.innerobj, _params); } /************************************************************************* Same as MLPCreate0, but with one hidden layer (NHid neurons) with non-linear activation function. Output layer is linear. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreate1(int nin, int nhid, int nout, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreate1(nin, nhid, nout, network.innerobj, null); } public static void mlpcreate1(int nin, int nhid, int nout, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreate1(nin, nhid, nout, network.innerobj, _params); } /************************************************************************* Same as MLPCreate0, but with two hidden layers (NHid1 and NHid2 neurons) with non-linear activation function. Output layer is linear. $ALL -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreate2(int nin, int nhid1, int nhid2, int nout, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreate2(nin, nhid1, nhid2, nout, network.innerobj, null); } public static void mlpcreate2(int nin, int nhid1, int nhid2, int nout, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreate2(nin, nhid1, nhid2, nout, network.innerobj, _params); } /************************************************************************* Creates neural network with NIn inputs, NOut outputs, without hidden layers with non-linear output layer. Network weights are filled with small random values. Activation function of the output layer takes values: (B, +INF), if D>=0 or (-INF, B), if D<0. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreateb0(int nin, int nout, double b, double d, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreateb0(nin, nout, b, d, network.innerobj, null); } public static void mlpcreateb0(int nin, int nout, double b, double d, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreateb0(nin, nout, b, d, network.innerobj, _params); } /************************************************************************* Same as MLPCreateB0 but with non-linear hidden layer. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreateb1(int nin, int nhid, int nout, double b, double d, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreateb1(nin, nhid, nout, b, d, network.innerobj, null); } public static void mlpcreateb1(int nin, int nhid, int nout, double b, double d, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreateb1(nin, nhid, nout, b, d, network.innerobj, _params); } /************************************************************************* Same as MLPCreateB0 but with two non-linear hidden layers. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreateb2(int nin, int nhid1, int nhid2, int nout, double b, double d, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreateb2(nin, nhid1, nhid2, nout, b, d, network.innerobj, null); } public static void mlpcreateb2(int nin, int nhid1, int nhid2, int nout, double b, double d, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreateb2(nin, nhid1, nhid2, nout, b, d, network.innerobj, _params); } /************************************************************************* Creates neural network with NIn inputs, NOut outputs, without hidden layers with non-linear output layer. Network weights are filled with small random values. Activation function of the output layer takes values [A,B]. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreater0(int nin, int nout, double a, double b, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreater0(nin, nout, a, b, network.innerobj, null); } public static void mlpcreater0(int nin, int nout, double a, double b, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreater0(nin, nout, a, b, network.innerobj, _params); } /************************************************************************* Same as MLPCreateR0, but with non-linear hidden layer. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreater1(int nin, int nhid, int nout, double a, double b, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreater1(nin, nhid, nout, a, b, network.innerobj, null); } public static void mlpcreater1(int nin, int nhid, int nout, double a, double b, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreater1(nin, nhid, nout, a, b, network.innerobj, _params); } /************************************************************************* Same as MLPCreateR0, but with two non-linear hidden layers. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreater2(int nin, int nhid1, int nhid2, int nout, double a, double b, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreater2(nin, nhid1, nhid2, nout, a, b, network.innerobj, null); } public static void mlpcreater2(int nin, int nhid1, int nhid2, int nout, double a, double b, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreater2(nin, nhid1, nhid2, nout, a, b, network.innerobj, _params); } /************************************************************************* Creates classifier network with NIn inputs and NOut possible classes. Network contains no hidden layers and linear output layer with SOFTMAX- normalization (so outputs sums up to 1.0 and converge to posterior probabilities). -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatec0(int nin, int nout, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreatec0(nin, nout, network.innerobj, null); } public static void mlpcreatec0(int nin, int nout, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreatec0(nin, nout, network.innerobj, _params); } /************************************************************************* Same as MLPCreateC0, but with one non-linear hidden layer. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatec1(int nin, int nhid, int nout, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreatec1(nin, nhid, nout, network.innerobj, null); } public static void mlpcreatec1(int nin, int nhid, int nout, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreatec1(nin, nhid, nout, network.innerobj, _params); } /************************************************************************* Same as MLPCreateC0, but with two non-linear hidden layers. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatec2(int nin, int nhid1, int nhid2, int nout, out multilayerperceptron network) { network = new multilayerperceptron(); mlpbase.mlpcreatec2(nin, nhid1, nhid2, nout, network.innerobj, null); } public static void mlpcreatec2(int nin, int nhid1, int nhid2, int nout, out multilayerperceptron network, alglib.xparams _params) { network = new multilayerperceptron(); mlpbase.mlpcreatec2(nin, nhid1, nhid2, nout, network.innerobj, _params); } /************************************************************************* Copying of neural network INPUT PARAMETERS: Network1 - original OUTPUT PARAMETERS: Network2 - copy -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcopy(multilayerperceptron network1, out multilayerperceptron network2) { network2 = new multilayerperceptron(); mlpbase.mlpcopy(network1.innerobj, network2.innerobj, null); } public static void mlpcopy(multilayerperceptron network1, out multilayerperceptron network2, alglib.xparams _params) { network2 = new multilayerperceptron(); mlpbase.mlpcopy(network1.innerobj, network2.innerobj, _params); } /************************************************************************* This function copies tunable parameters (weights/means/sigmas) from one network to another with same architecture. It performs some rudimentary checks that architectures are same, and throws exception if check fails. It is intended for fast copying of states between two network which are known to have same geometry. INPUT PARAMETERS: Network1 - source, must be correctly initialized Network2 - target, must have same architecture OUTPUT PARAMETERS: Network2 - network state is copied from source to target -- ALGLIB -- Copyright 20.06.2013 by Bochkanov Sergey *************************************************************************/ public static void mlpcopytunableparameters(multilayerperceptron network1, multilayerperceptron network2) { mlpbase.mlpcopytunableparameters(network1.innerobj, network2.innerobj, null); } public static void mlpcopytunableparameters(multilayerperceptron network1, multilayerperceptron network2, alglib.xparams _params) { mlpbase.mlpcopytunableparameters(network1.innerobj, network2.innerobj, _params); } /************************************************************************* Randomization of neural network weights -- ALGLIB -- Copyright 06.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlprandomize(multilayerperceptron network) { mlpbase.mlprandomize(network.innerobj, null); } public static void mlprandomize(multilayerperceptron network, alglib.xparams _params) { mlpbase.mlprandomize(network.innerobj, _params); } /************************************************************************* Randomization of neural network weights and standartisator -- ALGLIB -- Copyright 10.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlprandomizefull(multilayerperceptron network) { mlpbase.mlprandomizefull(network.innerobj, null); } public static void mlprandomizefull(multilayerperceptron network, alglib.xparams _params) { mlpbase.mlprandomizefull(network.innerobj, _params); } /************************************************************************* Internal subroutine. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpinitpreprocessor(multilayerperceptron network, double[,] xy, int ssize) { mlpbase.mlpinitpreprocessor(network.innerobj, xy, ssize, null); } public static void mlpinitpreprocessor(multilayerperceptron network, double[,] xy, int ssize, alglib.xparams _params) { mlpbase.mlpinitpreprocessor(network.innerobj, xy, ssize, _params); } /************************************************************************* Returns information about initialized network: number of inputs, outputs, weights. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpproperties(multilayerperceptron network, out int nin, out int nout, out int wcount) { nin = 0; nout = 0; wcount = 0; mlpbase.mlpproperties(network.innerobj, ref nin, ref nout, ref wcount, null); } public static void mlpproperties(multilayerperceptron network, out int nin, out int nout, out int wcount, alglib.xparams _params) { nin = 0; nout = 0; wcount = 0; mlpbase.mlpproperties(network.innerobj, ref nin, ref nout, ref wcount, _params); } /************************************************************************* Returns number of inputs. -- ALGLIB -- Copyright 19.10.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetinputscount(multilayerperceptron network) { return mlpbase.mlpgetinputscount(network.innerobj, null); } public static int mlpgetinputscount(multilayerperceptron network, alglib.xparams _params) { return mlpbase.mlpgetinputscount(network.innerobj, _params); } /************************************************************************* Returns number of outputs. -- ALGLIB -- Copyright 19.10.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetoutputscount(multilayerperceptron network) { return mlpbase.mlpgetoutputscount(network.innerobj, null); } public static int mlpgetoutputscount(multilayerperceptron network, alglib.xparams _params) { return mlpbase.mlpgetoutputscount(network.innerobj, _params); } /************************************************************************* Returns number of weights. -- ALGLIB -- Copyright 19.10.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetweightscount(multilayerperceptron network) { return mlpbase.mlpgetweightscount(network.innerobj, null); } public static int mlpgetweightscount(multilayerperceptron network, alglib.xparams _params) { return mlpbase.mlpgetweightscount(network.innerobj, _params); } /************************************************************************* Tells whether network is SOFTMAX-normalized (i.e. classifier) or not. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static bool mlpissoftmax(multilayerperceptron network) { return mlpbase.mlpissoftmax(network.innerobj, null); } public static bool mlpissoftmax(multilayerperceptron network, alglib.xparams _params) { return mlpbase.mlpissoftmax(network.innerobj, _params); } /************************************************************************* This function returns total number of layers (including input, hidden and output layers). -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetlayerscount(multilayerperceptron network) { return mlpbase.mlpgetlayerscount(network.innerobj, null); } public static int mlpgetlayerscount(multilayerperceptron network, alglib.xparams _params) { return mlpbase.mlpgetlayerscount(network.innerobj, _params); } /************************************************************************* This function returns size of K-th layer. K=0 corresponds to input layer, K=CNT-1 corresponds to output layer. Size of the output layer is always equal to the number of outputs, although when we have softmax-normalized network, last neuron doesn't have any connections - it is just zero. -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetlayersize(multilayerperceptron network, int k) { return mlpbase.mlpgetlayersize(network.innerobj, k, null); } public static int mlpgetlayersize(multilayerperceptron network, int k, alglib.xparams _params) { return mlpbase.mlpgetlayersize(network.innerobj, k, _params); } /************************************************************************* This function returns offset/scaling coefficients for I-th input of the network. INPUT PARAMETERS: Network - network I - input index OUTPUT PARAMETERS: Mean - mean term Sigma - sigma term, guaranteed to be nonzero. I-th input is passed through linear transformation IN[i] = (IN[i]-Mean)/Sigma before feeding to the network -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpgetinputscaling(multilayerperceptron network, int i, out double mean, out double sigma) { mean = 0; sigma = 0; mlpbase.mlpgetinputscaling(network.innerobj, i, ref mean, ref sigma, null); } public static void mlpgetinputscaling(multilayerperceptron network, int i, out double mean, out double sigma, alglib.xparams _params) { mean = 0; sigma = 0; mlpbase.mlpgetinputscaling(network.innerobj, i, ref mean, ref sigma, _params); } /************************************************************************* This function returns offset/scaling coefficients for I-th output of the network. INPUT PARAMETERS: Network - network I - input index OUTPUT PARAMETERS: Mean - mean term Sigma - sigma term, guaranteed to be nonzero. I-th output is passed through linear transformation OUT[i] = OUT[i]*Sigma+Mean before returning it to user. In case we have SOFTMAX-normalized network, we return (Mean,Sigma)=(0.0,1.0). -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpgetoutputscaling(multilayerperceptron network, int i, out double mean, out double sigma) { mean = 0; sigma = 0; mlpbase.mlpgetoutputscaling(network.innerobj, i, ref mean, ref sigma, null); } public static void mlpgetoutputscaling(multilayerperceptron network, int i, out double mean, out double sigma, alglib.xparams _params) { mean = 0; sigma = 0; mlpbase.mlpgetoutputscaling(network.innerobj, i, ref mean, ref sigma, _params); } /************************************************************************* This function returns information about Ith neuron of Kth layer INPUT PARAMETERS: Network - network K - layer index I - neuron index (within layer) OUTPUT PARAMETERS: FKind - activation function type (used by MLPActivationFunction()) this value is zero for input or linear neurons Threshold - also called offset, bias zero for input neurons NOTE: this function throws exception if layer or neuron with given index do not exists. -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpgetneuroninfo(multilayerperceptron network, int k, int i, out int fkind, out double threshold) { fkind = 0; threshold = 0; mlpbase.mlpgetneuroninfo(network.innerobj, k, i, ref fkind, ref threshold, null); } public static void mlpgetneuroninfo(multilayerperceptron network, int k, int i, out int fkind, out double threshold, alglib.xparams _params) { fkind = 0; threshold = 0; mlpbase.mlpgetneuroninfo(network.innerobj, k, i, ref fkind, ref threshold, _params); } /************************************************************************* This function returns information about connection from I0-th neuron of K0-th layer to I1-th neuron of K1-th layer. INPUT PARAMETERS: Network - network K0 - layer index I0 - neuron index (within layer) K1 - layer index I1 - neuron index (within layer) RESULT: connection weight (zero for non-existent connections) This function: 1. throws exception if layer or neuron with given index do not exists. 2. returns zero if neurons exist, but there is no connection between them -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static double mlpgetweight(multilayerperceptron network, int k0, int i0, int k1, int i1) { return mlpbase.mlpgetweight(network.innerobj, k0, i0, k1, i1, null); } public static double mlpgetweight(multilayerperceptron network, int k0, int i0, int k1, int i1, alglib.xparams _params) { return mlpbase.mlpgetweight(network.innerobj, k0, i0, k1, i1, _params); } /************************************************************************* This function sets offset/scaling coefficients for I-th input of the network. INPUT PARAMETERS: Network - network I - input index Mean - mean term Sigma - sigma term (if zero, will be replaced by 1.0) NTE: I-th input is passed through linear transformation IN[i] = (IN[i]-Mean)/Sigma before feeding to the network. This function sets Mean and Sigma. -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpsetinputscaling(multilayerperceptron network, int i, double mean, double sigma) { mlpbase.mlpsetinputscaling(network.innerobj, i, mean, sigma, null); } public static void mlpsetinputscaling(multilayerperceptron network, int i, double mean, double sigma, alglib.xparams _params) { mlpbase.mlpsetinputscaling(network.innerobj, i, mean, sigma, _params); } /************************************************************************* This function sets offset/scaling coefficients for I-th output of the network. INPUT PARAMETERS: Network - network I - input index Mean - mean term Sigma - sigma term (if zero, will be replaced by 1.0) OUTPUT PARAMETERS: NOTE: I-th output is passed through linear transformation OUT[i] = OUT[i]*Sigma+Mean before returning it to user. This function sets Sigma/Mean. In case we have SOFTMAX-normalized network, you can not set (Sigma,Mean) to anything other than(0.0,1.0) - this function will throw exception. -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpsetoutputscaling(multilayerperceptron network, int i, double mean, double sigma) { mlpbase.mlpsetoutputscaling(network.innerobj, i, mean, sigma, null); } public static void mlpsetoutputscaling(multilayerperceptron network, int i, double mean, double sigma, alglib.xparams _params) { mlpbase.mlpsetoutputscaling(network.innerobj, i, mean, sigma, _params); } /************************************************************************* This function modifies information about Ith neuron of Kth layer INPUT PARAMETERS: Network - network K - layer index I - neuron index (within layer) FKind - activation function type (used by MLPActivationFunction()) this value must be zero for input neurons (you can not set activation function for input neurons) Threshold - also called offset, bias this value must be zero for input neurons (you can not set threshold for input neurons) NOTES: 1. this function throws exception if layer or neuron with given index do not exists. 2. this function also throws exception when you try to set non-linear activation function for input neurons (any kind of network) or for output neurons of classifier network. 3. this function throws exception when you try to set non-zero threshold for input neurons (any kind of network). -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpsetneuroninfo(multilayerperceptron network, int k, int i, int fkind, double threshold) { mlpbase.mlpsetneuroninfo(network.innerobj, k, i, fkind, threshold, null); } public static void mlpsetneuroninfo(multilayerperceptron network, int k, int i, int fkind, double threshold, alglib.xparams _params) { mlpbase.mlpsetneuroninfo(network.innerobj, k, i, fkind, threshold, _params); } /************************************************************************* This function modifies information about connection from I0-th neuron of K0-th layer to I1-th neuron of K1-th layer. INPUT PARAMETERS: Network - network K0 - layer index I0 - neuron index (within layer) K1 - layer index I1 - neuron index (within layer) W - connection weight (must be zero for non-existent connections) This function: 1. throws exception if layer or neuron with given index do not exists. 2. throws exception if you try to set non-zero weight for non-existent connection -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpsetweight(multilayerperceptron network, int k0, int i0, int k1, int i1, double w) { mlpbase.mlpsetweight(network.innerobj, k0, i0, k1, i1, w, null); } public static void mlpsetweight(multilayerperceptron network, int k0, int i0, int k1, int i1, double w, alglib.xparams _params) { mlpbase.mlpsetweight(network.innerobj, k0, i0, k1, i1, w, _params); } /************************************************************************* Neural network activation function INPUT PARAMETERS: NET - neuron input K - function index (zero for linear function) OUTPUT PARAMETERS: F - function DF - its derivative D2F - its second derivative -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpactivationfunction(double net, int k, out double f, out double df, out double d2f) { f = 0; df = 0; d2f = 0; mlpbase.mlpactivationfunction(net, k, ref f, ref df, ref d2f, null); } public static void mlpactivationfunction(double net, int k, out double f, out double df, out double d2f, alglib.xparams _params) { f = 0; df = 0; d2f = 0; mlpbase.mlpactivationfunction(net, k, ref f, ref df, ref d2f, _params); } /************************************************************************* Procesing INPUT PARAMETERS: Network - neural network X - input vector, array[0..NIn-1]. OUTPUT PARAMETERS: Y - result. Regression estimate when solving regression task, vector of posterior probabilities for classification task. See also MLPProcessI -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpprocess(multilayerperceptron network, double[] x, ref double[] y) { mlpbase.mlpprocess(network.innerobj, x, ref y, null); } public static void mlpprocess(multilayerperceptron network, double[] x, ref double[] y, alglib.xparams _params) { mlpbase.mlpprocess(network.innerobj, x, ref y, _params); } /************************************************************************* 'interactive' variant of MLPProcess for languages like Python which support constructs like "Y = MLPProcess(NN,X)" and interactive mode of the interpreter This function allocates new array on each call, so it is significantly slower than its 'non-interactive' counterpart, but it is more convenient when you call it from command line. -- ALGLIB -- Copyright 21.09.2010 by Bochkanov Sergey *************************************************************************/ public static void mlpprocessi(multilayerperceptron network, double[] x, out double[] y) { y = new double[0]; mlpbase.mlpprocessi(network.innerobj, x, ref y, null); } public static void mlpprocessi(multilayerperceptron network, double[] x, out double[] y, alglib.xparams _params) { y = new double[0]; mlpbase.mlpprocessi(network.innerobj, x, ref y, _params); } /************************************************************************* Error of the neural network on dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static double mlperror(multilayerperceptron network, double[,] xy, int npoints) { return mlpbase.mlperror(network.innerobj, xy, npoints, null); } public static double mlperror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { return mlpbase.mlperror(network.innerobj, xy, npoints, _params); } /************************************************************************* Error of the neural network on dataset given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0 RESULT: sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static double mlperrorsparse(multilayerperceptron network, sparsematrix xy, int npoints) { return mlpbase.mlperrorsparse(network.innerobj, xy.innerobj, npoints, null); } public static double mlperrorsparse(multilayerperceptron network, sparsematrix xy, int npoints, alglib.xparams _params) { return mlpbase.mlperrorsparse(network.innerobj, xy.innerobj, npoints, _params); } /************************************************************************* Natural error function for neural network, internal subroutine. NOTE: this function is single-threaded. Unlike other error function, it receives no speed-up from being executed in SMP mode. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static double mlperrorn(multilayerperceptron network, double[,] xy, int ssize) { return mlpbase.mlperrorn(network.innerobj, xy, ssize, null); } public static double mlperrorn(multilayerperceptron network, double[,] xy, int ssize, alglib.xparams _params) { return mlpbase.mlperrorn(network.innerobj, xy, ssize, _params); } /************************************************************************* Classification error of the neural network on dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: classification error (number of misclassified cases) DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static int mlpclserror(multilayerperceptron network, double[,] xy, int npoints) { return mlpbase.mlpclserror(network.innerobj, xy, npoints, null); } public static int mlpclserror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { return mlpbase.mlpclserror(network.innerobj, xy, npoints, _params); } /************************************************************************* Relative classification error on the test set. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: Percent of incorrectly classified cases. Works both for classifier networks and general purpose networks used as classifiers. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 25.12.2008 by Bochkanov Sergey *************************************************************************/ public static double mlprelclserror(multilayerperceptron network, double[,] xy, int npoints) { return mlpbase.mlprelclserror(network.innerobj, xy, npoints, null); } public static double mlprelclserror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { return mlpbase.mlprelclserror(network.innerobj, xy, npoints, _params); } /************************************************************************* Relative classification error on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: Percent of incorrectly classified cases. Works both for classifier networks and general purpose networks used as classifiers. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 09.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlprelclserrorsparse(multilayerperceptron network, sparsematrix xy, int npoints) { return mlpbase.mlprelclserrorsparse(network.innerobj, xy.innerobj, npoints, null); } public static double mlprelclserrorsparse(multilayerperceptron network, sparsematrix xy, int npoints, alglib.xparams _params) { return mlpbase.mlprelclserrorsparse(network.innerobj, xy.innerobj, npoints, _params); } /************************************************************************* Average cross-entropy (in bits per element) on the test set. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: CrossEntropy/(NPoints*LN(2)). Zero if network solves regression task. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 08.01.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpavgce(multilayerperceptron network, double[,] xy, int npoints) { return mlpbase.mlpavgce(network.innerobj, xy, npoints, null); } public static double mlpavgce(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { return mlpbase.mlpavgce(network.innerobj, xy, npoints, _params); } /************************************************************************* Average cross-entropy (in bits per element) on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: CrossEntropy/(NPoints*LN(2)). Zero if network solves regression task. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 9.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlpavgcesparse(multilayerperceptron network, sparsematrix xy, int npoints) { return mlpbase.mlpavgcesparse(network.innerobj, xy.innerobj, npoints, null); } public static double mlpavgcesparse(multilayerperceptron network, sparsematrix xy, int npoints, alglib.xparams _params) { return mlpbase.mlpavgcesparse(network.innerobj, xy.innerobj, npoints, _params); } /************************************************************************* RMS error on the test set given. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: Root mean square error. Its meaning for regression task is obvious. As for classification task, RMS error means error when estimating posterior probabilities. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static double mlprmserror(multilayerperceptron network, double[,] xy, int npoints) { return mlpbase.mlprmserror(network.innerobj, xy, npoints, null); } public static double mlprmserror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { return mlpbase.mlprmserror(network.innerobj, xy, npoints, _params); } /************************************************************************* RMS error on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: Root mean square error. Its meaning for regression task is obvious. As for classification task, RMS error means error when estimating posterior probabilities. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 09.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlprmserrorsparse(multilayerperceptron network, sparsematrix xy, int npoints) { return mlpbase.mlprmserrorsparse(network.innerobj, xy.innerobj, npoints, null); } public static double mlprmserrorsparse(multilayerperceptron network, sparsematrix xy, int npoints, alglib.xparams _params) { return mlpbase.mlprmserrorsparse(network.innerobj, xy.innerobj, npoints, _params); } /************************************************************************* Average absolute error on the test set. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: Its meaning for regression task is obvious. As for classification task, it means average error when estimating posterior probabilities. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 11.03.2008 by Bochkanov Sergey *************************************************************************/ public static double mlpavgerror(multilayerperceptron network, double[,] xy, int npoints) { return mlpbase.mlpavgerror(network.innerobj, xy, npoints, null); } public static double mlpavgerror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { return mlpbase.mlpavgerror(network.innerobj, xy, npoints, _params); } /************************************************************************* Average absolute error on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: Its meaning for regression task is obvious. As for classification task, it means average error when estimating posterior probabilities. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 09.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlpavgerrorsparse(multilayerperceptron network, sparsematrix xy, int npoints) { return mlpbase.mlpavgerrorsparse(network.innerobj, xy.innerobj, npoints, null); } public static double mlpavgerrorsparse(multilayerperceptron network, sparsematrix xy, int npoints, alglib.xparams _params) { return mlpbase.mlpavgerrorsparse(network.innerobj, xy.innerobj, npoints, _params); } /************************************************************************* Average relative error on the test set. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: Its meaning for regression task is obvious. As for classification task, it means average relative error when estimating posterior probability of belonging to the correct class. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 11.03.2008 by Bochkanov Sergey *************************************************************************/ public static double mlpavgrelerror(multilayerperceptron network, double[,] xy, int npoints) { return mlpbase.mlpavgrelerror(network.innerobj, xy, npoints, null); } public static double mlpavgrelerror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { return mlpbase.mlpavgrelerror(network.innerobj, xy, npoints, _params); } /************************************************************************* Average relative error on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: Its meaning for regression task is obvious. As for classification task, it means average relative error when estimating posterior probability of belonging to the correct class. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 09.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlpavgrelerrorsparse(multilayerperceptron network, sparsematrix xy, int npoints) { return mlpbase.mlpavgrelerrorsparse(network.innerobj, xy.innerobj, npoints, null); } public static double mlpavgrelerrorsparse(multilayerperceptron network, sparsematrix xy, int npoints, alglib.xparams _params) { return mlpbase.mlpavgrelerrorsparse(network.innerobj, xy.innerobj, npoints, _params); } /************************************************************************* Gradient calculation INPUT PARAMETERS: Network - network initialized with one of the network creation funcs X - input vector, length of array must be at least NIn DesiredY- desired outputs, length of array must be at least NOut Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpgrad(multilayerperceptron network, double[] x, double[] desiredy, out double e, ref double[] grad) { e = 0; mlpbase.mlpgrad(network.innerobj, x, desiredy, ref e, ref grad, null); } public static void mlpgrad(multilayerperceptron network, double[] x, double[] desiredy, out double e, ref double[] grad, alglib.xparams _params) { e = 0; mlpbase.mlpgrad(network.innerobj, x, desiredy, ref e, ref grad, _params); } /************************************************************************* Gradient calculation (natural error function is used) INPUT PARAMETERS: Network - network initialized with one of the network creation funcs X - input vector, length of array must be at least NIn DesiredY- desired outputs, length of array must be at least NOut Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, sum-of-squares for regression networks, cross-entropy for classification networks. Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpgradn(multilayerperceptron network, double[] x, double[] desiredy, out double e, ref double[] grad) { e = 0; mlpbase.mlpgradn(network.innerobj, x, desiredy, ref e, ref grad, null); } public static void mlpgradn(multilayerperceptron network, double[] x, double[] desiredy, out double e, ref double[] grad, alglib.xparams _params) { e = 0; mlpbase.mlpgradn(network.innerobj, x, desiredy, ref e, ref grad, _params); } /************************************************************************* Batch gradient calculation for a set of inputs/outputs ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset in dense format; one sample = one row: * first NIn columns contain inputs, * for regression problem, next NOut columns store desired outputs. * for classification problem, next column (just one!) stores class number. SSize - number of elements in XY Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpgradbatch(multilayerperceptron network, double[,] xy, int ssize, out double e, ref double[] grad) { e = 0; mlpbase.mlpgradbatch(network.innerobj, xy, ssize, ref e, ref grad, null); } public static void mlpgradbatch(multilayerperceptron network, double[,] xy, int ssize, out double e, ref double[] grad, alglib.xparams _params) { e = 0; mlpbase.mlpgradbatch(network.innerobj, xy, ssize, ref e, ref grad, _params); } /************************************************************************* Batch gradient calculation for a set of inputs/outputs given by sparse matrices ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset in sparse format; one sample = one row: * MATRIX MUST BE STORED IN CRS FORMAT * first NIn columns contain inputs. * for regression problem, next NOut columns store desired outputs. * for classification problem, next column (just one!) stores class number. SSize - number of elements in XY Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpgradbatchsparse(multilayerperceptron network, sparsematrix xy, int ssize, out double e, ref double[] grad) { e = 0; mlpbase.mlpgradbatchsparse(network.innerobj, xy.innerobj, ssize, ref e, ref grad, null); } public static void mlpgradbatchsparse(multilayerperceptron network, sparsematrix xy, int ssize, out double e, ref double[] grad, alglib.xparams _params) { e = 0; mlpbase.mlpgradbatchsparse(network.innerobj, xy.innerobj, ssize, ref e, ref grad, _params); } /************************************************************************* Batch gradient calculation for a subset of dataset ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset in dense format; one sample = one row: * first NIn columns contain inputs, * for regression problem, next NOut columns store desired outputs. * for classification problem, next column (just one!) stores class number. SetSize - real size of XY, SetSize>=0; Idx - subset of SubsetSize elements, array[SubsetSize]: * Idx[I] stores row index in the original dataset which is given by XY. Gradient is calculated with respect to rows whose indexes are stored in Idx[]. * Idx[] must store correct indexes; this function throws an exception in case incorrect index (less than 0 or larger than rows(XY)) is given * Idx[] may store indexes in any order and even with repetitions. SubsetSize- number of elements in Idx[] array: * positive value means that subset given by Idx[] is processed * zero value results in zero gradient * negative value means that full dataset is processed Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpgradbatchsubset(multilayerperceptron network, double[,] xy, int setsize, int[] idx, int subsetsize, out double e, ref double[] grad) { e = 0; mlpbase.mlpgradbatchsubset(network.innerobj, xy, setsize, idx, subsetsize, ref e, ref grad, null); } public static void mlpgradbatchsubset(multilayerperceptron network, double[,] xy, int setsize, int[] idx, int subsetsize, out double e, ref double[] grad, alglib.xparams _params) { e = 0; mlpbase.mlpgradbatchsubset(network.innerobj, xy, setsize, idx, subsetsize, ref e, ref grad, _params); } /************************************************************************* Batch gradient calculation for a set of inputs/outputs for a subset of dataset given by set of indexes. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset in sparse format; one sample = one row: * MATRIX MUST BE STORED IN CRS FORMAT * first NIn columns contain inputs, * for regression problem, next NOut columns store desired outputs. * for classification problem, next column (just one!) stores class number. SetSize - real size of XY, SetSize>=0; Idx - subset of SubsetSize elements, array[SubsetSize]: * Idx[I] stores row index in the original dataset which is given by XY. Gradient is calculated with respect to rows whose indexes are stored in Idx[]. * Idx[] must store correct indexes; this function throws an exception in case incorrect index (less than 0 or larger than rows(XY)) is given * Idx[] may store indexes in any order and even with repetitions. SubsetSize- number of elements in Idx[] array: * positive value means that subset given by Idx[] is processed * zero value results in zero gradient * negative value means that full dataset is processed Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] NOTE: when SubsetSize<0 is used full dataset by call MLPGradBatchSparse function. -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpgradbatchsparsesubset(multilayerperceptron network, sparsematrix xy, int setsize, int[] idx, int subsetsize, out double e, ref double[] grad) { e = 0; mlpbase.mlpgradbatchsparsesubset(network.innerobj, xy.innerobj, setsize, idx, subsetsize, ref e, ref grad, null); } public static void mlpgradbatchsparsesubset(multilayerperceptron network, sparsematrix xy, int setsize, int[] idx, int subsetsize, out double e, ref double[] grad, alglib.xparams _params) { e = 0; mlpbase.mlpgradbatchsparsesubset(network.innerobj, xy.innerobj, setsize, idx, subsetsize, ref e, ref grad, _params); } /************************************************************************* Batch gradient calculation for a set of inputs/outputs (natural error function is used) INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - set of inputs/outputs; one sample = one row; first NIn columns contain inputs, next NOut columns - desired outputs. SSize - number of elements in XY Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, sum-of-squares for regression networks, cross-entropy for classification networks. Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpgradnbatch(multilayerperceptron network, double[,] xy, int ssize, out double e, ref double[] grad) { e = 0; mlpbase.mlpgradnbatch(network.innerobj, xy, ssize, ref e, ref grad, null); } public static void mlpgradnbatch(multilayerperceptron network, double[,] xy, int ssize, out double e, ref double[] grad, alglib.xparams _params) { e = 0; mlpbase.mlpgradnbatch(network.innerobj, xy, ssize, ref e, ref grad, _params); } /************************************************************************* Batch Hessian calculation (natural error function) using R-algorithm. Internal subroutine. -- ALGLIB -- Copyright 26.01.2008 by Bochkanov Sergey. Hessian calculation based on R-algorithm described in "Fast Exact Multiplication by the Hessian", B. A. Pearlmutter, Neural Computation, 1994. *************************************************************************/ public static void mlphessiannbatch(multilayerperceptron network, double[,] xy, int ssize, out double e, ref double[] grad, ref double[,] h) { e = 0; mlpbase.mlphessiannbatch(network.innerobj, xy, ssize, ref e, ref grad, ref h, null); } public static void mlphessiannbatch(multilayerperceptron network, double[,] xy, int ssize, out double e, ref double[] grad, ref double[,] h, alglib.xparams _params) { e = 0; mlpbase.mlphessiannbatch(network.innerobj, xy, ssize, ref e, ref grad, ref h, _params); } /************************************************************************* Batch Hessian calculation using R-algorithm. Internal subroutine. -- ALGLIB -- Copyright 26.01.2008 by Bochkanov Sergey. Hessian calculation based on R-algorithm described in "Fast Exact Multiplication by the Hessian", B. A. Pearlmutter, Neural Computation, 1994. *************************************************************************/ public static void mlphessianbatch(multilayerperceptron network, double[,] xy, int ssize, out double e, ref double[] grad, ref double[,] h) { e = 0; mlpbase.mlphessianbatch(network.innerobj, xy, ssize, ref e, ref grad, ref h, null); } public static void mlphessianbatch(multilayerperceptron network, double[,] xy, int ssize, out double e, ref double[] grad, ref double[,] h, alglib.xparams _params) { e = 0; mlpbase.mlphessianbatch(network.innerobj, xy, ssize, ref e, ref grad, ref h, _params); } /************************************************************************* Calculation of all types of errors on subset of dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset; one sample = one row; first NIn columns contain inputs, next NOut columns - desired outputs. SetSize - real size of XY, SetSize>=0; Subset - subset of SubsetSize elements, array[SubsetSize]; SubsetSize- number of elements in Subset[] array: * if SubsetSize>0, rows of XY with indices Subset[0]... ...Subset[SubsetSize-1] are processed * if SubsetSize=0, zeros are returned * if SubsetSize<0, entire dataset is processed; Subset[] array is ignored in this case. OUTPUT PARAMETERS: Rep - it contains all type of errors. -- ALGLIB -- Copyright 04.09.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpallerrorssubset(multilayerperceptron network, double[,] xy, int setsize, int[] subset, int subsetsize, out modelerrors rep) { rep = new modelerrors(); mlpbase.mlpallerrorssubset(network.innerobj, xy, setsize, subset, subsetsize, rep.innerobj, null); } public static void mlpallerrorssubset(multilayerperceptron network, double[,] xy, int setsize, int[] subset, int subsetsize, out modelerrors rep, alglib.xparams _params) { rep = new modelerrors(); mlpbase.mlpallerrorssubset(network.innerobj, xy, setsize, subset, subsetsize, rep.innerobj, _params); } /************************************************************************* Calculation of all types of errors on subset of dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset given by sparse matrix; one sample = one row; first NIn columns contain inputs, next NOut columns - desired outputs. SetSize - real size of XY, SetSize>=0; Subset - subset of SubsetSize elements, array[SubsetSize]; SubsetSize- number of elements in Subset[] array: * if SubsetSize>0, rows of XY with indices Subset[0]... ...Subset[SubsetSize-1] are processed * if SubsetSize=0, zeros are returned * if SubsetSize<0, entire dataset is processed; Subset[] array is ignored in this case. OUTPUT PARAMETERS: Rep - it contains all type of errors. -- ALGLIB -- Copyright 04.09.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpallerrorssparsesubset(multilayerperceptron network, sparsematrix xy, int setsize, int[] subset, int subsetsize, out modelerrors rep) { rep = new modelerrors(); mlpbase.mlpallerrorssparsesubset(network.innerobj, xy.innerobj, setsize, subset, subsetsize, rep.innerobj, null); } public static void mlpallerrorssparsesubset(multilayerperceptron network, sparsematrix xy, int setsize, int[] subset, int subsetsize, out modelerrors rep, alglib.xparams _params) { rep = new modelerrors(); mlpbase.mlpallerrorssparsesubset(network.innerobj, xy.innerobj, setsize, subset, subsetsize, rep.innerobj, _params); } /************************************************************************* Error of the neural network on subset of dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; SetSize - real size of XY, SetSize>=0; Subset - subset of SubsetSize elements, array[SubsetSize]; SubsetSize- number of elements in Subset[] array: * if SubsetSize>0, rows of XY with indices Subset[0]... ...Subset[SubsetSize-1] are processed * if SubsetSize=0, zeros are returned * if SubsetSize<0, entire dataset is processed; Subset[] array is ignored in this case. RESULT: sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 04.09.2012 by Bochkanov Sergey *************************************************************************/ public static double mlperrorsubset(multilayerperceptron network, double[,] xy, int setsize, int[] subset, int subsetsize) { return mlpbase.mlperrorsubset(network.innerobj, xy, setsize, subset, subsetsize, null); } public static double mlperrorsubset(multilayerperceptron network, double[,] xy, int setsize, int[] subset, int subsetsize, alglib.xparams _params) { return mlpbase.mlperrorsubset(network.innerobj, xy, setsize, subset, subsetsize, _params); } /************************************************************************* Error of the neural network on subset of sparse dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. SetSize - real size of XY, SetSize>=0; it is used when SubsetSize<0; Subset - subset of SubsetSize elements, array[SubsetSize]; SubsetSize- number of elements in Subset[] array: * if SubsetSize>0, rows of XY with indices Subset[0]... ...Subset[SubsetSize-1] are processed * if SubsetSize=0, zeros are returned * if SubsetSize<0, entire dataset is processed; Subset[] array is ignored in this case. RESULT: sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 04.09.2012 by Bochkanov Sergey *************************************************************************/ public static double mlperrorsparsesubset(multilayerperceptron network, sparsematrix xy, int setsize, int[] subset, int subsetsize) { return mlpbase.mlperrorsparsesubset(network.innerobj, xy.innerobj, setsize, subset, subsetsize, null); } public static double mlperrorsparsesubset(multilayerperceptron network, sparsematrix xy, int setsize, int[] subset, int subsetsize, alglib.xparams _params) { return mlpbase.mlperrorsparsesubset(network.innerobj, xy.innerobj, setsize, subset, subsetsize, _params); } } public partial class alglib { /************************************************************************* Multiclass Fisher LDA Subroutine finds coefficients of linear combination which optimally separates training set on classes. COMMERCIAL EDITION OF ALGLIB: ! Commercial version of ALGLIB includes two important improvements of ! this function, which can be used from C++ and C#: ! * Intel MKL support (lightweight Intel MKL is shipped with ALGLIB) ! * multithreading support ! ! Intel MKL gives approximately constant (with respect to number of ! worker threads) acceleration factor which depends on CPU being used, ! problem size and "baseline" ALGLIB edition which is used for ! comparison. Best results are achieved for high-dimensional problems ! (NVars is at least 256). ! ! Multithreading is used to accelerate initial phase of LDA, which ! includes calculation of products of large matrices. Again, for best ! efficiency problem must be high-dimensional. ! ! Generally, commercial ALGLIB is several times faster than open-source ! generic C edition, and many times faster than open-source C# edition. ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: XY - training set, array[0..NPoints-1,0..NVars]. First NVars columns store values of independent variables, next column stores number of class (from 0 to NClasses-1) which dataset element belongs to. Fractional values are rounded to nearest integer. NPoints - training set size, NPoints>=0 NVars - number of independent variables, NVars>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: Info - return code: * -4, if internal EVD subroutine hasn't converged * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, NVars<1, NClasses<2) * 1, if task has been solved * 2, if there was a multicollinearity in training set, but task has been solved. W - linear combination coefficients, array[0..NVars-1] -- ALGLIB -- Copyright 31.05.2008 by Bochkanov Sergey *************************************************************************/ public static void fisherlda(double[,] xy, int npoints, int nvars, int nclasses, out int info, out double[] w) { info = 0; w = new double[0]; lda.fisherlda(xy, npoints, nvars, nclasses, ref info, ref w, null); } public static void fisherlda(double[,] xy, int npoints, int nvars, int nclasses, out int info, out double[] w, alglib.xparams _params) { info = 0; w = new double[0]; lda.fisherlda(xy, npoints, nvars, nclasses, ref info, ref w, _params); } /************************************************************************* N-dimensional multiclass Fisher LDA Subroutine finds coefficients of linear combinations which optimally separates training set on classes. It returns N-dimensional basis whose vector are sorted by quality of training set separation (in descending order). ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: XY - training set, array[0..NPoints-1,0..NVars]. First NVars columns store values of independent variables, next column stores number of class (from 0 to NClasses-1) which dataset element belongs to. Fractional values are rounded to nearest integer. NPoints - training set size, NPoints>=0 NVars - number of independent variables, NVars>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: Info - return code: * -4, if internal EVD subroutine hasn't converged * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, NVars<1, NClasses<2) * 1, if task has been solved * 2, if there was a multicollinearity in training set, but task has been solved. W - basis, array[0..NVars-1,0..NVars-1] columns of matrix stores basis vectors, sorted by quality of training set separation (in descending order) -- ALGLIB -- Copyright 31.05.2008 by Bochkanov Sergey *************************************************************************/ public static void fisherldan(double[,] xy, int npoints, int nvars, int nclasses, out int info, out double[,] w) { info = 0; w = new double[0,0]; lda.fisherldan(xy, npoints, nvars, nclasses, ref info, ref w, null); } public static void fisherldan(double[,] xy, int npoints, int nvars, int nclasses, out int info, out double[,] w, alglib.xparams _params) { info = 0; w = new double[0,0]; lda.fisherldan(xy, npoints, nvars, nclasses, ref info, ref w, _params); } } public partial class alglib { /************************************************************************* This object stores state of the SSA model. You should use ALGLIB functions to work with this object. *************************************************************************/ public class ssamodel : alglibobject { // // Public declarations // public ssamodel() { _innerobj = new ssa.ssamodel(); } public override alglib.alglibobject make_copy() { return new ssamodel((ssa.ssamodel)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private ssa.ssamodel _innerobj; public ssa.ssamodel innerobj { get { return _innerobj; } } public ssamodel(ssa.ssamodel obj) { _innerobj = obj; } } /************************************************************************* This function creates SSA model object. Right after creation model is in "dummy" mode - you can add data, but analyzing/prediction will return just zeros (it assumes that basis is empty). HOW TO USE SSA MODEL: 1. create model with ssacreate() 2. add data with one/many ssaaddsequence() calls 3. choose SSA algorithm with one of ssasetalgo...() functions: * ssasetalgotopkdirect() for direct one-run analysis * ssasetalgotopkrealtime() for algorithm optimized for many subsequent runs with warm-start capabilities * ssasetalgoprecomputed() for user-supplied basis 4. set window width with ssasetwindow() 5. perform one of the analysis-related activities: a) call ssagetbasis() to get basis b) call ssaanalyzelast() ssaanalyzesequence() or ssaanalyzelastwindow() to perform analysis (trend/noise separation) c) call one of the forecasting functions (ssaforecastlast() or ssaforecastsequence()) to perform prediction; alternatively, you can extract linear recurrence coefficients with ssagetlrr(). SSA analysis will be performed during first call to analysis-related function. SSA model is smart enough to track all changes in the dataset and model settings, to cache previously computed basis and to re-evaluate basis only when necessary. Additionally, if your setting involves constant stream of incoming data, you can perform quick update already calculated model with one of the incremental append-and-update functions: ssaappendpointandupdate() or ssaappendsequenceandupdate(). NOTE: steps (2), (3), (4) can be performed in arbitrary order. INPUT PARAMETERS: none OUTPUT PARAMETERS: S - structure which stores model state -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssacreate(out ssamodel s) { s = new ssamodel(); ssa.ssacreate(s.innerobj, null); } public static void ssacreate(out ssamodel s, alglib.xparams _params) { s = new ssamodel(); ssa.ssacreate(s.innerobj, _params); } /************************************************************************* This function sets window width for SSA model. You should call it before analysis phase. Default window width is 1 (not for real use). Special notes: * this function call can be performed at any moment before first call to analysis-related functions * changing window width invalidates internally stored basis; if you change window width AFTER you call analysis-related function, next analysis phase will require re-calculation of the basis according to current algorithm. * calling this function with exactly same window width as current one has no effect * if you specify window width larger than any data sequence stored in the model, analysis will return zero basis. INPUT PARAMETERS: S - SSA model created with ssacreate() WindowWidth - >=1, new window width OUTPUT PARAMETERS: S - SSA model, updated -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetwindow(ssamodel s, int windowwidth) { ssa.ssasetwindow(s.innerobj, windowwidth, null); } public static void ssasetwindow(ssamodel s, int windowwidth, alglib.xparams _params) { ssa.ssasetwindow(s.innerobj, windowwidth, _params); } /************************************************************************* This function sets seed which is used to initialize internal RNG when we make pseudorandom decisions on model updates. By default, deterministic seed is used - which results in same sequence of pseudorandom decisions every time you run SSA model. If you specify non- deterministic seed value, then SSA model may return slightly different results after each run. This function can be useful when you have several SSA models updated with sseappendpointandupdate() called with 01 means that delayed power-up is performed -- ALGLIB -- Copyright 03.11.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetpoweruplength(ssamodel s, int pwlen) { ssa.ssasetpoweruplength(s.innerobj, pwlen, null); } public static void ssasetpoweruplength(ssamodel s, int pwlen, alglib.xparams _params) { ssa.ssasetpoweruplength(s.innerobj, pwlen, _params); } /************************************************************************* This function sets memory limit of SSA analysis. Straightforward SSA with sequence length T and window width W needs O(T*W) memory. It is possible to reduce memory consumption by splitting task into smaller chunks. Thus function allows you to specify approximate memory limit (measured in double precision numbers used for buffers). Actual memory consumption will be comparable to the number specified by you. Default memory limit is 50.000.000 (400Mbytes) in current version. INPUT PARAMETERS: S - SSA model MemLimit- memory limit, >=0. Zero value means no limit. -- ALGLIB -- Copyright 20.12.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetmemorylimit(ssamodel s, int memlimit) { ssa.ssasetmemorylimit(s.innerobj, memlimit, null); } public static void ssasetmemorylimit(ssamodel s, int memlimit, alglib.xparams _params) { ssa.ssasetmemorylimit(s.innerobj, memlimit, _params); } /************************************************************************* This function adds data sequence to SSA model. Only single-dimensional sequences are supported. What is a sequences? Following definitions/requirements apply: * a sequence is an array of values measured in subsequent, equally separated time moments (ticks). * you may have many sequences in your dataset; say, one sequence may correspond to one trading session. * sequence length should be larger than current window length (shorter sequences will be ignored during analysis). * analysis is performed within a sequence; different sequences are NOT stacked together to produce one large contiguous stream of data. * analysis is performed for all sequences at once, i.e. same set of basis vectors is computed for all sequences INCREMENTAL ANALYSIS This function is non intended for incremental updates of previously found SSA basis. Calling it invalidates all previous analysis results (basis is reset and will be recalculated from zero during next analysis). If you want to perform incremental/real-time SSA, consider using following functions: * ssaappendpointandupdate() for appending one point * ssaappendsequenceandupdate() for appending new sequence INPUT PARAMETERS: S - SSA model created with ssacreate() X - array[N], data, can be larger (additional values are ignored) N - data length, can be automatically determined from the array length. N>=0. OUTPUT PARAMETERS: S - SSA model, updated NOTE: you can clear dataset with ssacleardata() -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaaddsequence(ssamodel s, double[] x, int n) { ssa.ssaaddsequence(s.innerobj, x, n, null); } public static void ssaaddsequence(ssamodel s, double[] x, int n, alglib.xparams _params) { ssa.ssaaddsequence(s.innerobj, x, n, _params); } public static void ssaaddsequence(ssamodel s, double[] x) { int n; n = ap.len(x); ssa.ssaaddsequence(s.innerobj, x, n, null); return; } public static void ssaaddsequence(ssamodel s, double[] x, alglib.xparams _params) { int n; n = ap.len(x); ssa.ssaaddsequence(s.innerobj, x, n, _params); return; } /************************************************************************* This function appends single point to last data sequence stored in the SSA model and tries to update model in the incremental manner (if possible with current algorithm). If you want to add more than one point at once: * if you want to add M points to the same sequence, perform M-1 calls with UpdateIts parameter set to 0.0, and last call with non-zero UpdateIts. * if you want to add new sequence, use ssaappendsequenceandupdate() Running time of this function does NOT depend on dataset size, only on window width and number of singular vectors. Depending on algorithm being used, incremental update has complexity: * for top-K real time - O(UpdateIts*K*Width^2), with fractional UpdateIts * for top-K direct - O(Width^3) for any non-zero UpdateIts * for precomputed basis - O(1), no update is performed INPUT PARAMETERS: S - SSA model created with ssacreate() X - new point UpdateIts - >=0, floating point (!) value, desired update frequency: * zero value means that point is stored, but no update is performed * integer part of the value means that specified number of iterations is always performed * fractional part of the value means that one iteration is performed with this probability. Recommended value: 0=1, number of ticks in the sequence UpdateIts - >=0, floating point (!) value, desired update frequency: * zero value means that point is stored, but no update is performed * integer part of the value means that specified number of iterations is always performed * fractional part of the value means that one iteration is performed with this probability. Recommended value: 0=1 NBasis - number of basis vectors, 1<=NBasis<=WindowWidth OUTPUT PARAMETERS: S - updated model NOTE: calling this function invalidates basis in all cases. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetalgoprecomputed(ssamodel s, double[,] a, int windowwidth, int nbasis) { ssa.ssasetalgoprecomputed(s.innerobj, a, windowwidth, nbasis, null); } public static void ssasetalgoprecomputed(ssamodel s, double[,] a, int windowwidth, int nbasis, alglib.xparams _params) { ssa.ssasetalgoprecomputed(s.innerobj, a, windowwidth, nbasis, _params); } public static void ssasetalgoprecomputed(ssamodel s, double[,] a) { int windowwidth; int nbasis; windowwidth = ap.rows(a); nbasis = ap.cols(a); ssa.ssasetalgoprecomputed(s.innerobj, a, windowwidth, nbasis, null); return; } public static void ssasetalgoprecomputed(ssamodel s, double[,] a, alglib.xparams _params) { int windowwidth; int nbasis; windowwidth = ap.rows(a); nbasis = ap.cols(a); ssa.ssasetalgoprecomputed(s.innerobj, a, windowwidth, nbasis, _params); return; } /************************************************************************* This function sets SSA algorithm to "direct top-K" algorithm. "Direct top-K" algorithm performs full SVD of the N*WINDOW trajectory matrix (hence its name - direct solver is used), then extracts top K components. Overall running time is O(N*WINDOW^2), where N is a number of ticks in the dataset, WINDOW is window width. This algorithm may handle "append" requests which add just one/few ticks to the end of the last sequence in O(WINDOW^3) time, which is ~N/WINDOW times faster than re-computing everything from scratch. INPUT PARAMETERS: S - SSA model TopK - number of components to analyze; TopK>=1. OUTPUT PARAMETERS: S - updated model NOTE: TopK>WindowWidth is silently decreased to WindowWidth during analysis phase NOTE: calling this function invalidates basis, except for the situation when this algorithm was already set with same parameters. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetalgotopkdirect(ssamodel s, int topk) { ssa.ssasetalgotopkdirect(s.innerobj, topk, null); } public static void ssasetalgotopkdirect(ssamodel s, int topk, alglib.xparams _params) { ssa.ssasetalgotopkdirect(s.innerobj, topk, _params); } /************************************************************************* This function sets SSA algorithm to "top-K real time algorithm". This algo extracts K components with largest singular values. It is real-time version of top-K algorithm which is optimized for incremental processing and fast start-up. Internally it uses subspace eigensolver for truncated SVD. It results in ability to perform quick updates of the basis when only a few points/sequences is added to dataset. Performance profile of the algorithm is given below: * O(K*WindowWidth^2) running time for incremental update of the dataset with one of the "append-and-update" functions (ssaappendpointandupdate() or ssaappendsequenceandupdate()). * O(N*WindowWidth^2) running time for initial basis evaluation (N=size of dataset) * ability to split costly initialization across several incremental updates of the basis (so called "Power-Up" functionality, activated by ssasetpoweruplength() function) INPUT PARAMETERS: S - SSA model TopK - number of components to analyze; TopK>=1. OUTPUT PARAMETERS: S - updated model NOTE: this algorithm is optimized for large-scale tasks with large datasets. On toy problems with just 5-10 points it can return basis which is slightly different from that returned by direct algorithm (ssasetalgotopkdirect() function). However, the difference becomes negligible as dataset grows. NOTE: TopK>WindowWidth is silently decreased to WindowWidth during analysis phase NOTE: calling this function invalidates basis, except for the situation when this algorithm was already set with same parameters. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetalgotopkrealtime(ssamodel s, int topk) { ssa.ssasetalgotopkrealtime(s.innerobj, topk, null); } public static void ssasetalgotopkrealtime(ssamodel s, int topk, alglib.xparams _params) { ssa.ssasetalgotopkrealtime(s.innerobj, topk, _params); } /************************************************************************* This function clears all data stored in the model and invalidates all basis components found so far. INPUT PARAMETERS: S - SSA model created with ssacreate() OUTPUT PARAMETERS: S - SSA model, updated -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssacleardata(ssamodel s) { ssa.ssacleardata(s.innerobj, null); } public static void ssacleardata(ssamodel s, alglib.xparams _params) { ssa.ssacleardata(s.innerobj, _params); } /************************************************************************* This function executes SSA on internally stored dataset and returns basis found by current method. INPUT PARAMETERS: S - SSA model OUTPUT PARAMETERS: A - array[WindowWidth,NBasis], basis; vectors are stored in matrix columns, by descreasing variance SV - array[NBasis]: * zeros - for model initialized with SSASetAlgoPrecomputed() * singular values - for other algorithms WindowWidth - current window NBasis - basis size CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Calling this function in degenerate cases (no data or all data are shorter than window size; no algorithm is specified) returns basis with just one zero vector. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssagetbasis(ssamodel s, out double[,] a, out double[] sv, out int windowwidth, out int nbasis) { a = new double[0,0]; sv = new double[0]; windowwidth = 0; nbasis = 0; ssa.ssagetbasis(s.innerobj, ref a, ref sv, ref windowwidth, ref nbasis, null); } public static void ssagetbasis(ssamodel s, out double[,] a, out double[] sv, out int windowwidth, out int nbasis, alglib.xparams _params) { a = new double[0,0]; sv = new double[0]; windowwidth = 0; nbasis = 0; ssa.ssagetbasis(s.innerobj, ref a, ref sv, ref windowwidth, ref nbasis, _params); } /************************************************************************* This function returns linear recurrence relation (LRR) coefficients found by current SSA algorithm. INPUT PARAMETERS: S - SSA model OUTPUT PARAMETERS: A - array[WindowWidth-1]. Coefficients of the linear recurrence of the form: X[W-1] = X[W-2]*A[W-2] + X[W-3]*A[W-3] + ... + X[0]*A[0]. Empty array for WindowWidth=1. WindowWidth - current window width CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Calling this function in degenerate cases (no data or all data are shorter than window size; no algorithm is specified) returns zeros. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssagetlrr(ssamodel s, out double[] a, out int windowwidth) { a = new double[0]; windowwidth = 0; ssa.ssagetlrr(s.innerobj, ref a, ref windowwidth, null); } public static void ssagetlrr(ssamodel s, out double[] a, out int windowwidth, alglib.xparams _params) { a = new double[0]; windowwidth = 0; ssa.ssagetlrr(s.innerobj, ref a, ref windowwidth, _params); } /************************************************************************* This function executes SSA on internally stored dataset and returns analysis for the last window of the last sequence. Such analysis is an lightweight alternative for full scale reconstruction (see below). Typical use case for this function is real-time setting, when you are interested in quick-and-dirty (very quick and very dirty) processing of just a few last ticks of the trend. IMPORTANT: full scale SSA involves analysis of the ENTIRE dataset, with reconstruction being done for all positions of sliding window with subsequent hankelization (diagonal averaging) of the resulting matrix. Such analysis requires O((DataLen-Window)*Window*NBasis) FLOPs and can be quite costly. However, it has nice noise-canceling effects due to averaging. This function performs REDUCED analysis of the last window. It is much faster - just O(Window*NBasis), but its results are DIFFERENT from that of ssaanalyzelast(). In particular, first few points of the trend are much more prone to noise. INPUT PARAMETERS: S - SSA model OUTPUT PARAMETERS: Trend - array[WindowSize], reconstructed trend line Noise - array[WindowSize], the rest of the signal; it holds that ActualData = Trend+Noise. NTicks - current WindowSize CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. In any case, only basis is reused. Reconstruction is performed from scratch every time you call this function. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * last sequence is shorter than the window length (analysis can be done, but we can not perform reconstruction on the last sequence) Calling this function in degenerate cases returns following result: * in any case, WindowWidth ticks is returned * trend is assumed to be zero * noise is initialized by the last sequence; if last sequence is shorter than the window size, it is moved to the end of the array, and the beginning of the noise array is filled by zeros No analysis is performed in degenerate cases (we immediately return dummy values, no basis is constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaanalyzelastwindow(ssamodel s, out double[] trend, out double[] noise, out int nticks) { trend = new double[0]; noise = new double[0]; nticks = 0; ssa.ssaanalyzelastwindow(s.innerobj, ref trend, ref noise, ref nticks, null); } public static void ssaanalyzelastwindow(ssamodel s, out double[] trend, out double[] noise, out int nticks, alglib.xparams _params) { trend = new double[0]; noise = new double[0]; nticks = 0; ssa.ssaanalyzelastwindow(s.innerobj, ref trend, ref noise, ref nticks, _params); } /************************************************************************* This function: * builds SSA basis using internally stored (entire) dataset * returns reconstruction for the last NTicks of the last sequence If you want to analyze some other sequence, use ssaanalyzesequence(). Reconstruction phase involves generation of NTicks-WindowWidth sliding windows, their decomposition using empirical orthogonal functions found by SSA, followed by averaging of each data point across several overlapping windows. Thus, every point in the output trend is reconstructed using up to WindowWidth overlapping windows (WindowWidth windows exactly in the inner points, just one window at the extremal points). IMPORTANT: due to averaging this function returns different results for different values of NTicks. It is expected and not a bug. For example: * Trend[NTicks-1] is always same because it is not averaged in any case (same applies to Trend[0]). * Trend[NTicks-2] has different values for NTicks=WindowWidth and NTicks=WindowWidth+1 because former case means that no averaging is performed, and latter case means that averaging using two sliding windows is performed. Larger values of NTicks produce same results as NTicks=WindowWidth+1. * ...and so on... PERFORMANCE: this function has O((NTicks-WindowWidth)*WindowWidth*NBasis) running time. If you work in time-constrained setting and have to analyze just a few last ticks, choosing NTicks equal to WindowWidth+SmoothingLen, with SmoothingLen=1...WindowWidth will result in good compromise between noise cancellation and analysis speed. INPUT PARAMETERS: S - SSA model NTicks - number of ticks to analyze, Nticks>=1. * special case of NTicks<=WindowWidth is handled by analyzing last window and returning NTicks last ticks. * special case NTicks>LastSequenceLen is handled by prepending result with NTicks-LastSequenceLen zeros. OUTPUT PARAMETERS: Trend - array[NTicks], reconstructed trend line Noise - array[NTicks], the rest of the signal; it holds that ActualData = Trend+Noise. CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. In any case, only basis is reused. Reconstruction is performed from scratch every time you call this function. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * last sequence is shorter than the window length (analysis can be done, but we can not perform reconstruction on the last sequence) Calling this function in degenerate cases returns following result: * in any case, NTicks ticks is returned * trend is assumed to be zero * noise is initialized by the last sequence; if last sequence is shorter than the window size, it is moved to the end of the array, and the beginning of the noise array is filled by zeros No analysis is performed in degenerate cases (we immediately return dummy values, no basis is constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaanalyzelast(ssamodel s, int nticks, out double[] trend, out double[] noise) { trend = new double[0]; noise = new double[0]; ssa.ssaanalyzelast(s.innerobj, nticks, ref trend, ref noise, null); } public static void ssaanalyzelast(ssamodel s, int nticks, out double[] trend, out double[] noise, alglib.xparams _params) { trend = new double[0]; noise = new double[0]; ssa.ssaanalyzelast(s.innerobj, nticks, ref trend, ref noise, _params); } /************************************************************************* This function: * builds SSA basis using internally stored (entire) dataset * returns reconstruction for the sequence being passed to this function If you want to analyze last sequence stored in the model, use ssaanalyzelast(). Reconstruction phase involves generation of NTicks-WindowWidth sliding windows, their decomposition using empirical orthogonal functions found by SSA, followed by averaging of each data point across several overlapping windows. Thus, every point in the output trend is reconstructed using up to WindowWidth overlapping windows (WindowWidth windows exactly in the inner points, just one window at the extremal points). PERFORMANCE: this function has O((NTicks-WindowWidth)*WindowWidth*NBasis) running time. If you work in time-constrained setting and have to analyze just a few last ticks, choosing NTicks equal to WindowWidth+SmoothingLen, with SmoothingLen=1...WindowWidth will result in good compromise between noise cancellation and analysis speed. INPUT PARAMETERS: S - SSA model Data - array[NTicks], can be larger (only NTicks leading elements will be used) NTicks - number of ticks to analyze, Nticks>=1. * special case of NTicks=1 OUTPUT PARAMETERS: Trend - array[NTicks], predicted trend line CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * last sequence is shorter than the WindowWidth (analysis can be done, but we can not perform forecasting on the last sequence) * window lentgh is 1 (impossible to use for forecasting) * SSA analysis algorithm is configured to extract basis whose size is equal to window length (impossible to use for forecasting; only basis whose size is less than window length can be used). Calling this function in degenerate cases returns following result: * NTicks copies of the last value is returned for non-empty task with large enough dataset, but with overcomplete basis (window width=1 or basis size is equal to window width) * zero trend with length=NTicks is returned for empty task No analysis is performed in degenerate cases (we immediately return dummy values, no basis is ever constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaforecastlast(ssamodel s, int nticks, out double[] trend) { trend = new double[0]; ssa.ssaforecastlast(s.innerobj, nticks, ref trend, null); } public static void ssaforecastlast(ssamodel s, int nticks, out double[] trend, alglib.xparams _params) { trend = new double[0]; ssa.ssaforecastlast(s.innerobj, nticks, ref trend, _params); } /************************************************************************* This function builds SSA basis and performs forecasting for a user- specified sequence, returning value of trend. Forecasting is done in two stages: * first, we extract trend from the WindowWidth last elements of the sequence. This stage is optional, you can turn it off if you pass data which are already processed with SSA. Of course, you can turn it off even for raw data, but it is not recommended - noise suppression is very important for correct prediction. * then, we apply LRR for last WindowWidth-1 elements of the extracted trend. This function has following running time: * O(NBasis*WindowWidth) for trend extraction phase * O(WindowWidth*NTicks) for forecast phase NOTE: this algorithm performs prediction using only one - last - sliding window. Predictions produced by such approach are smooth continuations of the reconstructed trend line, but they can be easily corrupted by noise. If you need noise-resistant prediction, use ssaforecastavgsequence() function, which averages predictions built using several sliding windows. INPUT PARAMETERS: S - SSA model Data - array[NTicks], data to forecast DataLen - number of ticks in the data, DataLen>=1 ForecastLen - number of ticks to predict, ForecastLen>=1 ApplySmoothing - whether to apply smoothing trend extraction or not; if you do not know what to specify, pass True. OUTPUT PARAMETERS: Trend - array[ForecastLen], forecasted trend CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * data sequence is shorter than the WindowWidth (analysis can be done, but we can not perform forecasting on the last sequence) * window lentgh is 1 (impossible to use for forecasting) * SSA analysis algorithm is configured to extract basis whose size is equal to window length (impossible to use for forecasting; only basis whose size is less than window length can be used). Calling this function in degenerate cases returns following result: * ForecastLen copies of the last value is returned for non-empty task with large enough dataset, but with overcomplete basis (window width=1 or basis size is equal to window width) * zero trend with length=ForecastLen is returned for empty task No analysis is performed in degenerate cases (we immediately return dummy values, no basis is ever constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaforecastsequence(ssamodel s, double[] data, int datalen, int forecastlen, bool applysmoothing, out double[] trend) { trend = new double[0]; ssa.ssaforecastsequence(s.innerobj, data, datalen, forecastlen, applysmoothing, ref trend, null); } public static void ssaforecastsequence(ssamodel s, double[] data, int datalen, int forecastlen, bool applysmoothing, out double[] trend, alglib.xparams _params) { trend = new double[0]; ssa.ssaforecastsequence(s.innerobj, data, datalen, forecastlen, applysmoothing, ref trend, _params); } public static void ssaforecastsequence(ssamodel s, double[] data, int forecastlen, out double[] trend) { int datalen; bool applysmoothing; trend = new double[0]; datalen = ap.len(data); applysmoothing = true; ssa.ssaforecastsequence(s.innerobj, data, datalen, forecastlen, applysmoothing, ref trend, null); return; } public static void ssaforecastsequence(ssamodel s, double[] data, int forecastlen, out double[] trend, alglib.xparams _params) { int datalen; bool applysmoothing; trend = new double[0]; datalen = ap.len(data); applysmoothing = true; ssa.ssaforecastsequence(s.innerobj, data, datalen, forecastlen, applysmoothing, ref trend, _params); return; } /************************************************************************* This function builds SSA basis and performs forecasting for a specified number of ticks, returning value of trend. Forecast is performed as follows: * SSA trend extraction is applied to last M sliding windows of the internally stored dataset * for each of M sliding windows, M predictions are built * average value of M predictions is returned This function has following running time: * O(NBasis*WindowWidth*M) for trend extraction phase (always performed) * O(WindowWidth*NTicks*M) for forecast phase NOTE: noise reduction is ALWAYS applied by this algorithm; if you want to apply recurrence relation to raw unprocessed data, use another function - ssaforecastsequence() which allows to turn on and off noise reduction phase. NOTE: combination of several predictions results in lesser sensitivity to noise, but it may produce undesirable discontinuities between last point of the trend and first point of the prediction. The reason is that last point of the trend is usually corrupted by noise, but average value of several predictions is less sensitive to noise, thus discontinuity appears. It is not a bug. INPUT PARAMETERS: S - SSA model M - number of sliding windows to combine, M>=1. If your dataset has less than M sliding windows, this parameter will be silently reduced. NTicks - number of ticks to forecast, NTicks>=1 OUTPUT PARAMETERS: Trend - array[NTicks], predicted trend line CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * last sequence is shorter than the WindowWidth (analysis can be done, but we can not perform forecasting on the last sequence) * window lentgh is 1 (impossible to use for forecasting) * SSA analysis algorithm is configured to extract basis whose size is equal to window length (impossible to use for forecasting; only basis whose size is less than window length can be used). Calling this function in degenerate cases returns following result: * NTicks copies of the last value is returned for non-empty task with large enough dataset, but with overcomplete basis (window width=1 or basis size is equal to window width) * zero trend with length=NTicks is returned for empty task No analysis is performed in degenerate cases (we immediately return dummy values, no basis is ever constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaforecastavglast(ssamodel s, int m, int nticks, out double[] trend) { trend = new double[0]; ssa.ssaforecastavglast(s.innerobj, m, nticks, ref trend, null); } public static void ssaforecastavglast(ssamodel s, int m, int nticks, out double[] trend, alglib.xparams _params) { trend = new double[0]; ssa.ssaforecastavglast(s.innerobj, m, nticks, ref trend, _params); } /************************************************************************* This function builds SSA basis and performs forecasting for a user- specified sequence, returning value of trend. Forecasting is done in two stages: * first, we extract trend from M last sliding windows of the sequence. This stage is optional, you can turn it off if you pass data which are already processed with SSA. Of course, you can turn it off even for raw data, but it is not recommended - noise suppression is very important for correct prediction. * then, we apply LRR independently for M sliding windows * average of M predictions is returned This function has following running time: * O(NBasis*WindowWidth*M) for trend extraction phase * O(WindowWidth*NTicks*M) for forecast phase NOTE: combination of several predictions results in lesser sensitivity to noise, but it may produce undesirable discontinuities between last point of the trend and first point of the prediction. The reason is that last point of the trend is usually corrupted by noise, but average value of several predictions is less sensitive to noise, thus discontinuity appears. It is not a bug. INPUT PARAMETERS: S - SSA model Data - array[NTicks], data to forecast DataLen - number of ticks in the data, DataLen>=1 M - number of sliding windows to combine, M>=1. If your dataset has less than M sliding windows, this parameter will be silently reduced. ForecastLen - number of ticks to predict, ForecastLen>=1 ApplySmoothing - whether to apply smoothing trend extraction or not. if you do not know what to specify, pass true. OUTPUT PARAMETERS: Trend - array[ForecastLen], forecasted trend CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * data sequence is shorter than the WindowWidth (analysis can be done, but we can not perform forecasting on the last sequence) * window lentgh is 1 (impossible to use for forecasting) * SSA analysis algorithm is configured to extract basis whose size is equal to window length (impossible to use for forecasting; only basis whose size is less than window length can be used). Calling this function in degenerate cases returns following result: * ForecastLen copies of the last value is returned for non-empty task with large enough dataset, but with overcomplete basis (window width=1 or basis size is equal to window width) * zero trend with length=ForecastLen is returned for empty task No analysis is performed in degenerate cases (we immediately return dummy values, no basis is ever constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaforecastavgsequence(ssamodel s, double[] data, int datalen, int m, int forecastlen, bool applysmoothing, out double[] trend) { trend = new double[0]; ssa.ssaforecastavgsequence(s.innerobj, data, datalen, m, forecastlen, applysmoothing, ref trend, null); } public static void ssaforecastavgsequence(ssamodel s, double[] data, int datalen, int m, int forecastlen, bool applysmoothing, out double[] trend, alglib.xparams _params) { trend = new double[0]; ssa.ssaforecastavgsequence(s.innerobj, data, datalen, m, forecastlen, applysmoothing, ref trend, _params); } public static void ssaforecastavgsequence(ssamodel s, double[] data, int m, int forecastlen, out double[] trend) { int datalen; bool applysmoothing; trend = new double[0]; datalen = ap.len(data); applysmoothing = true; ssa.ssaforecastavgsequence(s.innerobj, data, datalen, m, forecastlen, applysmoothing, ref trend, null); return; } public static void ssaforecastavgsequence(ssamodel s, double[] data, int m, int forecastlen, out double[] trend, alglib.xparams _params) { int datalen; bool applysmoothing; trend = new double[0]; datalen = ap.len(data); applysmoothing = true; ssa.ssaforecastavgsequence(s.innerobj, data, datalen, m, forecastlen, applysmoothing, ref trend, _params); return; } } public partial class alglib { /************************************************************************* *************************************************************************/ public class linearmodel : alglibobject { // // Public declarations // public linearmodel() { _innerobj = new linreg.linearmodel(); } public override alglib.alglibobject make_copy() { return new linearmodel((linreg.linearmodel)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private linreg.linearmodel _innerobj; public linreg.linearmodel innerobj { get { return _innerobj; } } public linearmodel(linreg.linearmodel obj) { _innerobj = obj; } } /************************************************************************* LRReport structure contains additional information about linear model: * C - covariation matrix, array[0..NVars,0..NVars]. C[i,j] = Cov(A[i],A[j]) * RMSError - root mean square error on a training set * AvgError - average error on a training set * AvgRelError - average relative error on a training set (excluding observations with zero function value). * CVRMSError - leave-one-out cross-validation estimate of generalization error. Calculated using fast algorithm with O(NVars*NPoints) complexity. * CVAvgError - cross-validation estimate of average error * CVAvgRelError - cross-validation estimate of average relative error All other fields of the structure are intended for internal use and should not be used outside ALGLIB. *************************************************************************/ public class lrreport : alglibobject { // // Public declarations // public double[,] c { get { return _innerobj.c; } set { _innerobj.c = value; } } public double rmserror { get { return _innerobj.rmserror; } set { _innerobj.rmserror = value; } } public double avgerror { get { return _innerobj.avgerror; } set { _innerobj.avgerror = value; } } public double avgrelerror { get { return _innerobj.avgrelerror; } set { _innerobj.avgrelerror = value; } } public double cvrmserror { get { return _innerobj.cvrmserror; } set { _innerobj.cvrmserror = value; } } public double cvavgerror { get { return _innerobj.cvavgerror; } set { _innerobj.cvavgerror = value; } } public double cvavgrelerror { get { return _innerobj.cvavgrelerror; } set { _innerobj.cvavgrelerror = value; } } public int ncvdefects { get { return _innerobj.ncvdefects; } set { _innerobj.ncvdefects = value; } } public int[] cvdefects { get { return _innerobj.cvdefects; } set { _innerobj.cvdefects = value; } } public lrreport() { _innerobj = new linreg.lrreport(); } public override alglib.alglibobject make_copy() { return new lrreport((linreg.lrreport)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private linreg.lrreport _innerobj; public linreg.lrreport innerobj { get { return _innerobj; } } public lrreport(linreg.lrreport obj) { _innerobj = obj; } } /************************************************************************* Linear regression Subroutine builds model: Y = A(0)*X[0] + ... + A(N-1)*X[N-1] + A(N) and model found in ALGLIB format, covariation matrix, training set errors (rms, average, average relative) and leave-one-out cross-validation estimate of the generalization error. CV estimate calculated using fast algorithm with O(NPoints*NVars) complexity. When covariation matrix is calculated standard deviations of function values are assumed to be equal to RMS error on the training set. INPUT PARAMETERS: XY - training set, array [0..NPoints-1,0..NVars]: * NVars columns - independent variables * last column - dependent variable NPoints - training set size, NPoints>NVars+1 NVars - number of independent variables OUTPUT PARAMETERS: Info - return code: * -255, in case of unknown internal error * -4, if internal SVD subroutine haven't converged * -1, if incorrect parameters was passed (NPoints0. NPoints - training set size, NPoints>NVars+1 NVars - number of independent variables OUTPUT PARAMETERS: Info - return code: * -255, in case of unknown internal error * -4, if internal SVD subroutine haven't converged * -1, if incorrect parameters was passed (NPoints=0 K - K>=1 (K can be larger than N , such cases will be correctly handled). Window width. K=1 corresponds to identity transformation (nothing changes). OUTPUT PARAMETERS: X - array, whose first N elements were processed with SMA(K) NOTE 1: this function uses efficient in-place algorithm which does not allocate temporary arrays. NOTE 2: this algorithm makes only one pass through array and uses running sum to speed-up calculation of the averages. Additional measures are taken to ensure that running sum on a long sequence of zero elements will be correctly reset to zero even in the presence of round-off error. NOTE 3: this is unsymmetric version of the algorithm, which does NOT averages points after the current one. Only X[i], X[i-1], ... are used when calculating new value of X[i]. We should also note that this algorithm uses BOTH previous points and current one, i.e. new value of X[i] depends on BOTH previous point and X[i] itself. -- ALGLIB -- Copyright 25.10.2011 by Bochkanov Sergey *************************************************************************/ public static void filtersma(ref double[] x, int n, int k) { filters.filtersma(ref x, n, k, null); } public static void filtersma(ref double[] x, int n, int k, alglib.xparams _params) { filters.filtersma(ref x, n, k, _params); } public static void filtersma(ref double[] x, int k) { int n; n = ap.len(x); filters.filtersma(ref x, n, k, null); return; } public static void filtersma(ref double[] x, int k, alglib.xparams _params) { int n; n = ap.len(x); filters.filtersma(ref x, n, k, _params); return; } /************************************************************************* Filters: exponential moving averages. This filter replaces array by results of EMA(alpha) filter. EMA(alpha) is defined as filter which replaces X[] by S[]: S[0] = X[0] S[t] = alpha*X[t] + (1-alpha)*S[t-1] INPUT PARAMETERS: X - array[N], array to process. It can be larger than N, in this case only first N points are processed. N - points count, N>=0 alpha - 0=0 K - K>=1 (K can be larger than N , such cases will be correctly handled). Window width. K=1 corresponds to identity transformation (nothing changes). OUTPUT PARAMETERS: X - array, whose first N elements were processed with SMA(K) NOTE 1: this function uses efficient in-place algorithm which does not allocate temporary arrays. NOTE 2: this algorithm makes only one pass through array and uses running sum to speed-up calculation of the averages. Additional measures are taken to ensure that running sum on a long sequence of zero elements will be correctly reset to zero even in the presence of round-off error. NOTE 3: this is unsymmetric version of the algorithm, which does NOT averages points after the current one. Only X[i], X[i-1], ... are used when calculating new value of X[i]. We should also note that this algorithm uses BOTH previous points and current one, i.e. new value of X[i] depends on BOTH previous point and X[i] itself. -- ALGLIB -- Copyright 25.10.2011 by Bochkanov Sergey *************************************************************************/ public static void filterlrma(ref double[] x, int n, int k) { filters.filterlrma(ref x, n, k, null); } public static void filterlrma(ref double[] x, int n, int k, alglib.xparams _params) { filters.filterlrma(ref x, n, k, _params); } public static void filterlrma(ref double[] x, int k) { int n; n = ap.len(x); filters.filterlrma(ref x, n, k, null); return; } public static void filterlrma(ref double[] x, int k, alglib.xparams _params) { int n; n = ap.len(x); filters.filterlrma(ref x, n, k, _params); return; } } public partial class alglib { /************************************************************************* *************************************************************************/ public class logitmodel : alglibobject { // // Public declarations // public logitmodel() { _innerobj = new logit.logitmodel(); } public override alglib.alglibobject make_copy() { return new logitmodel((logit.logitmodel)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private logit.logitmodel _innerobj; public logit.logitmodel innerobj { get { return _innerobj; } } public logitmodel(logit.logitmodel obj) { _innerobj = obj; } } /************************************************************************* MNLReport structure contains information about training process: * NGrad - number of gradient calculations * NHess - number of Hessian calculations *************************************************************************/ public class mnlreport : alglibobject { // // Public declarations // public int ngrad { get { return _innerobj.ngrad; } set { _innerobj.ngrad = value; } } public int nhess { get { return _innerobj.nhess; } set { _innerobj.nhess = value; } } public mnlreport() { _innerobj = new logit.mnlreport(); } public override alglib.alglibobject make_copy() { return new mnlreport((logit.mnlreport)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private logit.mnlreport _innerobj; public logit.mnlreport innerobj { get { return _innerobj; } } public mnlreport(logit.mnlreport obj) { _innerobj = obj; } } /************************************************************************* This subroutine trains logit model. INPUT PARAMETERS: XY - training set, array[0..NPoints-1,0..NVars] First NVars columns store values of independent variables, next column stores number of class (from 0 to NClasses-1) which dataset element belongs to. Fractional values are rounded to nearest integer. NPoints - training set size, NPoints>=1 NVars - number of independent variables, NVars>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: Info - return code: * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints=1 OUTPUT PARAMETERS: State - structure stores algorithm state -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdcreate(int n, out mcpdstate s) { s = new mcpdstate(); mcpd.mcpdcreate(n, s.innerobj, null); } public static void mcpdcreate(int n, out mcpdstate s, alglib.xparams _params) { s = new mcpdstate(); mcpd.mcpdcreate(n, s.innerobj, _params); } /************************************************************************* DESCRIPTION: This function is a specialized version of MCPDCreate() function, and we recommend you to read comments for this function for general information about MCPD solver. This function creates MCPD (Markov Chains for Population Data) solver for "Entry-state" model, i.e. model where transition from X[i] to X[i+1] is modelled as X[i+1] = P*X[i] where X[i] and X[i+1] are N-dimensional state vectors P is a N*N transition matrix and one selected component of X[] is called "entry" state and is treated in a special way: system state always transits from "entry" state to some another state system state can not transit from any state into "entry" state Such conditions basically mean that row of P which corresponds to "entry" state is zero. Such models arise when: * there is some population of individuals * individuals can have different states * individuals can transit from one state to another * population size is NOT constant - at every moment of time there is some (unpredictable) amount of "new" individuals, which can transit into one of the states at the next turn, but still no one leaves population * you want to model transitions of individuals from one state into another * but you do NOT want to predict amount of "new" individuals because it does not depends on individuals already present (hence system can not transit INTO entry state - it can only transit FROM it). This model is discussed in more details in the ALGLIB User Guide (see http://www.alglib.net/dataanalysis/ for more data). INPUT PARAMETERS: N - problem dimension, N>=2 EntryState- index of entry state, in 0..N-1 OUTPUT PARAMETERS: State - structure stores algorithm state -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdcreateentry(int n, int entrystate, out mcpdstate s) { s = new mcpdstate(); mcpd.mcpdcreateentry(n, entrystate, s.innerobj, null); } public static void mcpdcreateentry(int n, int entrystate, out mcpdstate s, alglib.xparams _params) { s = new mcpdstate(); mcpd.mcpdcreateentry(n, entrystate, s.innerobj, _params); } /************************************************************************* DESCRIPTION: This function is a specialized version of MCPDCreate() function, and we recommend you to read comments for this function for general information about MCPD solver. This function creates MCPD (Markov Chains for Population Data) solver for "Exit-state" model, i.e. model where transition from X[i] to X[i+1] is modelled as X[i+1] = P*X[i] where X[i] and X[i+1] are N-dimensional state vectors P is a N*N transition matrix and one selected component of X[] is called "exit" state and is treated in a special way: system state can transit from any state into "exit" state system state can not transit from "exit" state into any other state transition operator discards "exit" state (makes it zero at each turn) Such conditions basically mean that column of P which corresponds to "exit" state is zero. Multiplication by such P may decrease sum of vector components. Such models arise when: * there is some population of individuals * individuals can have different states * individuals can transit from one state to another * population size is NOT constant - individuals can move into "exit" state and leave population at the next turn, but there are no new individuals * amount of individuals which leave population can be predicted * you want to model transitions of individuals from one state into another (including transitions into the "exit" state) This model is discussed in more details in the ALGLIB User Guide (see http://www.alglib.net/dataanalysis/ for more data). INPUT PARAMETERS: N - problem dimension, N>=2 ExitState- index of exit state, in 0..N-1 OUTPUT PARAMETERS: State - structure stores algorithm state -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdcreateexit(int n, int exitstate, out mcpdstate s) { s = new mcpdstate(); mcpd.mcpdcreateexit(n, exitstate, s.innerobj, null); } public static void mcpdcreateexit(int n, int exitstate, out mcpdstate s, alglib.xparams _params) { s = new mcpdstate(); mcpd.mcpdcreateexit(n, exitstate, s.innerobj, _params); } /************************************************************************* DESCRIPTION: This function is a specialized version of MCPDCreate() function, and we recommend you to read comments for this function for general information about MCPD solver. This function creates MCPD (Markov Chains for Population Data) solver for "Entry-Exit-states" model, i.e. model where transition from X[i] to X[i+1] is modelled as X[i+1] = P*X[i] where X[i] and X[i+1] are N-dimensional state vectors P is a N*N transition matrix one selected component of X[] is called "entry" state and is treated in a special way: system state always transits from "entry" state to some another state system state can not transit from any state into "entry" state and another one component of X[] is called "exit" state and is treated in a special way too: system state can transit from any state into "exit" state system state can not transit from "exit" state into any other state transition operator discards "exit" state (makes it zero at each turn) Such conditions basically mean that: row of P which corresponds to "entry" state is zero column of P which corresponds to "exit" state is zero Multiplication by such P may decrease sum of vector components. Such models arise when: * there is some population of individuals * individuals can have different states * individuals can transit from one state to another * population size is NOT constant * at every moment of time there is some (unpredictable) amount of "new" individuals, which can transit into one of the states at the next turn * some individuals can move (predictably) into "exit" state and leave population at the next turn * you want to model transitions of individuals from one state into another, including transitions from the "entry" state and into the "exit" state. * but you do NOT want to predict amount of "new" individuals because it does not depends on individuals already present (hence system can not transit INTO entry state - it can only transit FROM it). This model is discussed in more details in the ALGLIB User Guide (see http://www.alglib.net/dataanalysis/ for more data). INPUT PARAMETERS: N - problem dimension, N>=2 EntryState- index of entry state, in 0..N-1 ExitState- index of exit state, in 0..N-1 OUTPUT PARAMETERS: State - structure stores algorithm state -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdcreateentryexit(int n, int entrystate, int exitstate, out mcpdstate s) { s = new mcpdstate(); mcpd.mcpdcreateentryexit(n, entrystate, exitstate, s.innerobj, null); } public static void mcpdcreateentryexit(int n, int entrystate, int exitstate, out mcpdstate s, alglib.xparams _params) { s = new mcpdstate(); mcpd.mcpdcreateentryexit(n, entrystate, exitstate, s.innerobj, _params); } /************************************************************************* This function is used to add a track - sequence of system states at the different moments of its evolution. You may add one or several tracks to the MCPD solver. In case you have several tracks, they won't overwrite each other. For example, if you pass two tracks, A1-A2-A3 (system at t=A+1, t=A+2 and t=A+3) and B1-B2-B3, then solver will try to model transitions from t=A+1 to t=A+2, t=A+2 to t=A+3, t=B+1 to t=B+2, t=B+2 to t=B+3. But it WONT mix these two tracks - i.e. it wont try to model transition from t=A+3 to t=B+1. INPUT PARAMETERS: S - solver XY - track, array[K,N]: * I-th row is a state at t=I * elements of XY must be non-negative (exception will be thrown on negative elements) K - number of points in a track * if given, only leading K rows of XY are used * if not given, automatically determined from size of XY NOTES: 1. Track may contain either proportional or population data: * with proportional data all rows of XY must sum to 1.0, i.e. we have proportions instead of absolute population values * with population data rows of XY contain population counts and generally do not sum to 1.0 (although they still must be non-negative) -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdaddtrack(mcpdstate s, double[,] xy, int k) { mcpd.mcpdaddtrack(s.innerobj, xy, k, null); } public static void mcpdaddtrack(mcpdstate s, double[,] xy, int k, alglib.xparams _params) { mcpd.mcpdaddtrack(s.innerobj, xy, k, _params); } public static void mcpdaddtrack(mcpdstate s, double[,] xy) { int k; k = ap.rows(xy); mcpd.mcpdaddtrack(s.innerobj, xy, k, null); return; } public static void mcpdaddtrack(mcpdstate s, double[,] xy, alglib.xparams _params) { int k; k = ap.rows(xy); mcpd.mcpdaddtrack(s.innerobj, xy, k, _params); return; } /************************************************************************* This function is used to add equality constraints on the elements of the transition matrix P. MCPD solver has four types of constraints which can be placed on P: * user-specified equality constraints (optional) * user-specified bound constraints (optional) * user-specified general linear constraints (optional) * basic constraints (always present): * non-negativity: P[i,j]>=0 * consistency: every column of P sums to 1.0 Final constraints which are passed to the underlying optimizer are calculated as intersection of all present constraints. For example, you may specify boundary constraint on P[0,0] and equality one: 0.1<=P[0,0]<=0.9 P[0,0]=0.5 Such combination of constraints will be silently reduced to their intersection, which is P[0,0]=0.5. This function can be used to place equality constraints on arbitrary subset of elements of P. Set of constraints is specified by EC, which may contain either NAN's or finite numbers from [0,1]. NAN denotes absence of constraint, finite number denotes equality constraint on specific element of P. You can also use MCPDAddEC() function which allows to ADD equality constraint for one element of P without changing constraints for other elements. These functions (MCPDSetEC and MCPDAddEC) interact as follows: * there is internal matrix of equality constraints which is stored in the MCPD solver * MCPDSetEC() replaces this matrix by another one (SET) * MCPDAddEC() modifies one element of this matrix and leaves other ones unchanged (ADD) * thus MCPDAddEC() call preserves all modifications done by previous calls, while MCPDSetEC() completely discards all changes done to the equality constraints. INPUT PARAMETERS: S - solver EC - equality constraints, array[N,N]. Elements of EC can be either NAN's or finite numbers from [0,1]. NAN denotes absence of constraints, while finite value denotes equality constraint on the corresponding element of P. NOTES: 1. infinite values of EC will lead to exception being thrown. Values less than 0.0 or greater than 1.0 will lead to error code being returned after call to MCPDSolve(). -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsetec(mcpdstate s, double[,] ec) { mcpd.mcpdsetec(s.innerobj, ec, null); } public static void mcpdsetec(mcpdstate s, double[,] ec, alglib.xparams _params) { mcpd.mcpdsetec(s.innerobj, ec, _params); } /************************************************************************* This function is used to add equality constraints on the elements of the transition matrix P. MCPD solver has four types of constraints which can be placed on P: * user-specified equality constraints (optional) * user-specified bound constraints (optional) * user-specified general linear constraints (optional) * basic constraints (always present): * non-negativity: P[i,j]>=0 * consistency: every column of P sums to 1.0 Final constraints which are passed to the underlying optimizer are calculated as intersection of all present constraints. For example, you may specify boundary constraint on P[0,0] and equality one: 0.1<=P[0,0]<=0.9 P[0,0]=0.5 Such combination of constraints will be silently reduced to their intersection, which is P[0,0]=0.5. This function can be used to ADD equality constraint for one element of P without changing constraints for other elements. You can also use MCPDSetEC() function which allows you to specify arbitrary set of equality constraints in one call. These functions (MCPDSetEC and MCPDAddEC) interact as follows: * there is internal matrix of equality constraints which is stored in the MCPD solver * MCPDSetEC() replaces this matrix by another one (SET) * MCPDAddEC() modifies one element of this matrix and leaves other ones unchanged (ADD) * thus MCPDAddEC() call preserves all modifications done by previous calls, while MCPDSetEC() completely discards all changes done to the equality constraints. INPUT PARAMETERS: S - solver I - row index of element being constrained J - column index of element being constrained C - value (constraint for P[I,J]). Can be either NAN (no constraint) or finite value from [0,1]. NOTES: 1. infinite values of C will lead to exception being thrown. Values less than 0.0 or greater than 1.0 will lead to error code being returned after call to MCPDSolve(). -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdaddec(mcpdstate s, int i, int j, double c) { mcpd.mcpdaddec(s.innerobj, i, j, c, null); } public static void mcpdaddec(mcpdstate s, int i, int j, double c, alglib.xparams _params) { mcpd.mcpdaddec(s.innerobj, i, j, c, _params); } /************************************************************************* This function is used to add bound constraints on the elements of the transition matrix P. MCPD solver has four types of constraints which can be placed on P: * user-specified equality constraints (optional) * user-specified bound constraints (optional) * user-specified general linear constraints (optional) * basic constraints (always present): * non-negativity: P[i,j]>=0 * consistency: every column of P sums to 1.0 Final constraints which are passed to the underlying optimizer are calculated as intersection of all present constraints. For example, you may specify boundary constraint on P[0,0] and equality one: 0.1<=P[0,0]<=0.9 P[0,0]=0.5 Such combination of constraints will be silently reduced to their intersection, which is P[0,0]=0.5. This function can be used to place bound constraints on arbitrary subset of elements of P. Set of constraints is specified by BndL/BndU matrices, which may contain arbitrary combination of finite numbers or infinities (like -INF=0 * consistency: every column of P sums to 1.0 Final constraints which are passed to the underlying optimizer are calculated as intersection of all present constraints. For example, you may specify boundary constraint on P[0,0] and equality one: 0.1<=P[0,0]<=0.9 P[0,0]=0.5 Such combination of constraints will be silently reduced to their intersection, which is P[0,0]=0.5. This function can be used to ADD bound constraint for one element of P without changing constraints for other elements. You can also use MCPDSetBC() function which allows to place bound constraints on arbitrary subset of elements of P. Set of constraints is specified by BndL/BndU matrices, which may contain arbitrary combination of finite numbers or infinities (like -INF=" (CT[i]>0). Your constraint may involve only some subset of P (less than N*N elements). For example it can be something like P[0,0] + P[0,1] = 0.5 In this case you still should pass matrix with N*N+1 columns, but all its elements (except for C[0,0], C[0,1] and C[0,N*N-1]) will be zero. INPUT PARAMETERS: S - solver C - array[K,N*N+1] - coefficients of constraints (see above for complete description) CT - array[K] - constraint types (see above for complete description) K - number of equality/inequality constraints, K>=0: * if given, only leading K elements of C/CT are used * if not given, automatically determined from sizes of C/CT -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsetlc(mcpdstate s, double[,] c, int[] ct, int k) { mcpd.mcpdsetlc(s.innerobj, c, ct, k, null); } public static void mcpdsetlc(mcpdstate s, double[,] c, int[] ct, int k, alglib.xparams _params) { mcpd.mcpdsetlc(s.innerobj, c, ct, k, _params); } public static void mcpdsetlc(mcpdstate s, double[,] c, int[] ct) { int k; if( (ap.rows(c)!=ap.len(ct))) throw new alglibexception("Error while calling 'mcpdsetlc': looks like one of arguments has wrong size"); k = ap.rows(c); mcpd.mcpdsetlc(s.innerobj, c, ct, k, null); return; } public static void mcpdsetlc(mcpdstate s, double[,] c, int[] ct, alglib.xparams _params) { int k; if( (ap.rows(c)!=ap.len(ct))) throw new alglibexception("Error while calling 'mcpdsetlc': looks like one of arguments has wrong size"); k = ap.rows(c); mcpd.mcpdsetlc(s.innerobj, c, ct, k, _params); return; } /************************************************************************* This function allows to tune amount of Tikhonov regularization being applied to your problem. By default, regularizing term is equal to r*||P-prior_P||^2, where r is a small non-zero value, P is transition matrix, prior_P is identity matrix, ||X||^2 is a sum of squared elements of X. This function allows you to change coefficient r. You can also change prior values with MCPDSetPrior() function. INPUT PARAMETERS: S - solver V - regularization coefficient, finite non-negative value. It is not recommended to specify zero value unless you are pretty sure that you want it. -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsettikhonovregularizer(mcpdstate s, double v) { mcpd.mcpdsettikhonovregularizer(s.innerobj, v, null); } public static void mcpdsettikhonovregularizer(mcpdstate s, double v, alglib.xparams _params) { mcpd.mcpdsettikhonovregularizer(s.innerobj, v, _params); } /************************************************************************* This function allows to set prior values used for regularization of your problem. By default, regularizing term is equal to r*||P-prior_P||^2, where r is a small non-zero value, P is transition matrix, prior_P is identity matrix, ||X||^2 is a sum of squared elements of X. This function allows you to change prior values prior_P. You can also change r with MCPDSetTikhonovRegularizer() function. INPUT PARAMETERS: S - solver PP - array[N,N], matrix of prior values: 1. elements must be real numbers from [0,1] 2. columns must sum to 1.0. First property is checked (exception is thrown otherwise), while second one is not checked/enforced. -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsetprior(mcpdstate s, double[,] pp) { mcpd.mcpdsetprior(s.innerobj, pp, null); } public static void mcpdsetprior(mcpdstate s, double[,] pp, alglib.xparams _params) { mcpd.mcpdsetprior(s.innerobj, pp, _params); } /************************************************************************* This function is used to change prediction weights MCPD solver scales prediction errors as follows Error(P) = ||W*(y-P*x)||^2 where x is a system state at time t y is a system state at time t+1 P is a transition matrix W is a diagonal scaling matrix By default, weights are chosen in order to minimize relative prediction error instead of absolute one. For example, if one component of state is about 0.5 in magnitude and another one is about 0.05, then algorithm will make corresponding weights equal to 2.0 and 20.0. INPUT PARAMETERS: S - solver PW - array[N], weights: * must be non-negative values (exception will be thrown otherwise) * zero values will be replaced by automatically chosen values -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsetpredictionweights(mcpdstate s, double[] pw) { mcpd.mcpdsetpredictionweights(s.innerobj, pw, null); } public static void mcpdsetpredictionweights(mcpdstate s, double[] pw, alglib.xparams _params) { mcpd.mcpdsetpredictionweights(s.innerobj, pw, _params); } /************************************************************************* This function is used to start solution of the MCPD problem. After return from this function, you can use MCPDResults() to get solution and completion code. -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsolve(mcpdstate s) { mcpd.mcpdsolve(s.innerobj, null); } public static void mcpdsolve(mcpdstate s, alglib.xparams _params) { mcpd.mcpdsolve(s.innerobj, _params); } /************************************************************************* MCPD results INPUT PARAMETERS: State - algorithm state OUTPUT PARAMETERS: P - array[N,N], transition matrix Rep - optimization report. You should check Rep.TerminationType in order to distinguish successful termination from unsuccessful one. Speaking short, positive values denote success, negative ones are failures. More information about fields of this structure can be found in the comments on MCPDReport datatype. -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdresults(mcpdstate s, out double[,] p, out mcpdreport rep) { p = new double[0,0]; rep = new mcpdreport(); mcpd.mcpdresults(s.innerobj, ref p, rep.innerobj, null); } public static void mcpdresults(mcpdstate s, out double[,] p, out mcpdreport rep, alglib.xparams _params) { p = new double[0,0]; rep = new mcpdreport(); mcpd.mcpdresults(s.innerobj, ref p, rep.innerobj, _params); } } public partial class alglib { /************************************************************************* Neural networks ensemble *************************************************************************/ public class mlpensemble : alglibobject { // // Public declarations // public mlpensemble() { _innerobj = new mlpe.mlpensemble(); } public override alglib.alglibobject make_copy() { return new mlpensemble((mlpe.mlpensemble)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private mlpe.mlpensemble _innerobj; public mlpe.mlpensemble innerobj { get { return _innerobj; } } public mlpensemble(mlpe.mlpensemble obj) { _innerobj = obj; } } /************************************************************************* This function serializes data structure to string. Important properties of s_out: * it contains alphanumeric characters, dots, underscores, minus signs * these symbols are grouped into words, which are separated by spaces and Windows-style (CR+LF) newlines * although serializer uses spaces and CR+LF as separators, you can replace any separator character by arbitrary combination of spaces, tabs, Windows or Unix newlines. It allows flexible reformatting of the string in case you want to include it into text or XML file. But you should not insert separators into the middle of the "words" nor you should change case of letters. * s_out can be freely moved between 32-bit and 64-bit systems, little and big endian machines, and so on. You can serialize structure on 32-bit machine and unserialize it on 64-bit one (or vice versa), or serialize it on SPARC and unserialize on x86. You can also serialize it in C# version of ALGLIB and unserialize in C++ one, and vice versa. *************************************************************************/ public static void mlpeserialize(mlpensemble obj, out string s_out) { alglib.serializer s = new alglib.serializer(); s.alloc_start(); mlpe.mlpealloc(s, obj.innerobj, null); s.sstart_str(); mlpe.mlpeserialize(s, obj.innerobj, null); s.stop(); s_out = s.get_string(); } /************************************************************************* This function unserializes data structure from string. *************************************************************************/ public static void mlpeunserialize(string s_in, out mlpensemble obj) { alglib.serializer s = new alglib.serializer(); obj = new mlpensemble(); s.ustart_str(s_in); mlpe.mlpeunserialize(s, obj.innerobj, null); s.stop(); } /************************************************************************* This function serializes data structure to stream. Data stream generated by this function is same as string representation generated by string version of serializer - alphanumeric characters, dots, underscores, minus signs, which are grouped into words separated by spaces and CR+LF. We recommend you to read comments on string version of serializer to find out more about serialization of AlGLIB objects. *************************************************************************/ public static void mlpeserialize(mlpensemble obj, System.IO.Stream stream_out) { alglib.serializer s = new alglib.serializer(); s.alloc_start(); mlpe.mlpealloc(s, obj.innerobj, null); s.sstart_stream(stream_out); mlpe.mlpeserialize(s, obj.innerobj, null); s.stop(); } /************************************************************************* This function unserializes data structure from stream. *************************************************************************/ public static void mlpeunserialize(System.IO.Stream stream_in, out mlpensemble obj) { alglib.serializer s = new alglib.serializer(); obj = new mlpensemble(); s.ustart_stream(stream_in); mlpe.mlpeunserialize(s, obj.innerobj, null); s.stop(); } /************************************************************************* Like MLPCreate0, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreate0(int nin, int nout, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreate0(nin, nout, ensemblesize, ensemble.innerobj, null); } public static void mlpecreate0(int nin, int nout, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreate0(nin, nout, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreate1, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreate1(int nin, int nhid, int nout, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreate1(nin, nhid, nout, ensemblesize, ensemble.innerobj, null); } public static void mlpecreate1(int nin, int nhid, int nout, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreate1(nin, nhid, nout, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreate2, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreate2(int nin, int nhid1, int nhid2, int nout, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreate2(nin, nhid1, nhid2, nout, ensemblesize, ensemble.innerobj, null); } public static void mlpecreate2(int nin, int nhid1, int nhid2, int nout, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreate2(nin, nhid1, nhid2, nout, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreateB0, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreateb0(int nin, int nout, double b, double d, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreateb0(nin, nout, b, d, ensemblesize, ensemble.innerobj, null); } public static void mlpecreateb0(int nin, int nout, double b, double d, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreateb0(nin, nout, b, d, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreateB1, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreateb1(int nin, int nhid, int nout, double b, double d, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreateb1(nin, nhid, nout, b, d, ensemblesize, ensemble.innerobj, null); } public static void mlpecreateb1(int nin, int nhid, int nout, double b, double d, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreateb1(nin, nhid, nout, b, d, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreateB2, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreateb2(int nin, int nhid1, int nhid2, int nout, double b, double d, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreateb2(nin, nhid1, nhid2, nout, b, d, ensemblesize, ensemble.innerobj, null); } public static void mlpecreateb2(int nin, int nhid1, int nhid2, int nout, double b, double d, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreateb2(nin, nhid1, nhid2, nout, b, d, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreateR0, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreater0(int nin, int nout, double a, double b, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreater0(nin, nout, a, b, ensemblesize, ensemble.innerobj, null); } public static void mlpecreater0(int nin, int nout, double a, double b, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreater0(nin, nout, a, b, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreateR1, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreater1(int nin, int nhid, int nout, double a, double b, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreater1(nin, nhid, nout, a, b, ensemblesize, ensemble.innerobj, null); } public static void mlpecreater1(int nin, int nhid, int nout, double a, double b, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreater1(nin, nhid, nout, a, b, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreateR2, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreater2(int nin, int nhid1, int nhid2, int nout, double a, double b, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreater2(nin, nhid1, nhid2, nout, a, b, ensemblesize, ensemble.innerobj, null); } public static void mlpecreater2(int nin, int nhid1, int nhid2, int nout, double a, double b, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreater2(nin, nhid1, nhid2, nout, a, b, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreateC0, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreatec0(int nin, int nout, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreatec0(nin, nout, ensemblesize, ensemble.innerobj, null); } public static void mlpecreatec0(int nin, int nout, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreatec0(nin, nout, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreateC1, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreatec1(int nin, int nhid, int nout, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreatec1(nin, nhid, nout, ensemblesize, ensemble.innerobj, null); } public static void mlpecreatec1(int nin, int nhid, int nout, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreatec1(nin, nhid, nout, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Like MLPCreateC2, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreatec2(int nin, int nhid1, int nhid2, int nout, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreatec2(nin, nhid1, nhid2, nout, ensemblesize, ensemble.innerobj, null); } public static void mlpecreatec2(int nin, int nhid1, int nhid2, int nout, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreatec2(nin, nhid1, nhid2, nout, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Creates ensemble from network. Only network geometry is copied. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreatefromnetwork(multilayerperceptron network, int ensemblesize, out mlpensemble ensemble) { ensemble = new mlpensemble(); mlpe.mlpecreatefromnetwork(network.innerobj, ensemblesize, ensemble.innerobj, null); } public static void mlpecreatefromnetwork(multilayerperceptron network, int ensemblesize, out mlpensemble ensemble, alglib.xparams _params) { ensemble = new mlpensemble(); mlpe.mlpecreatefromnetwork(network.innerobj, ensemblesize, ensemble.innerobj, _params); } /************************************************************************* Randomization of MLP ensemble -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlperandomize(mlpensemble ensemble) { mlpe.mlperandomize(ensemble.innerobj, null); } public static void mlperandomize(mlpensemble ensemble, alglib.xparams _params) { mlpe.mlperandomize(ensemble.innerobj, _params); } /************************************************************************* Return ensemble properties (number of inputs and outputs). -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpeproperties(mlpensemble ensemble, out int nin, out int nout) { nin = 0; nout = 0; mlpe.mlpeproperties(ensemble.innerobj, ref nin, ref nout, null); } public static void mlpeproperties(mlpensemble ensemble, out int nin, out int nout, alglib.xparams _params) { nin = 0; nout = 0; mlpe.mlpeproperties(ensemble.innerobj, ref nin, ref nout, _params); } /************************************************************************* Return normalization type (whether ensemble is SOFTMAX-normalized or not). -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static bool mlpeissoftmax(mlpensemble ensemble) { return mlpe.mlpeissoftmax(ensemble.innerobj, null); } public static bool mlpeissoftmax(mlpensemble ensemble, alglib.xparams _params) { return mlpe.mlpeissoftmax(ensemble.innerobj, _params); } /************************************************************************* Procesing INPUT PARAMETERS: Ensemble- neural networks ensemble X - input vector, array[0..NIn-1]. Y - (possibly) preallocated buffer; if size of Y is less than NOut, it will be reallocated. If it is large enough, it is NOT reallocated, so we can save some time on reallocation. OUTPUT PARAMETERS: Y - result. Regression estimate when solving regression task, vector of posterior probabilities for classification task. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpeprocess(mlpensemble ensemble, double[] x, ref double[] y) { mlpe.mlpeprocess(ensemble.innerobj, x, ref y, null); } public static void mlpeprocess(mlpensemble ensemble, double[] x, ref double[] y, alglib.xparams _params) { mlpe.mlpeprocess(ensemble.innerobj, x, ref y, _params); } /************************************************************************* 'interactive' variant of MLPEProcess for languages like Python which support constructs like "Y = MLPEProcess(LM,X)" and interactive mode of the interpreter This function allocates new array on each call, so it is significantly slower than its 'non-interactive' counterpart, but it is more convenient when you call it from command line. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpeprocessi(mlpensemble ensemble, double[] x, out double[] y) { y = new double[0]; mlpe.mlpeprocessi(ensemble.innerobj, x, ref y, null); } public static void mlpeprocessi(mlpensemble ensemble, double[] x, out double[] y, alglib.xparams _params) { y = new double[0]; mlpe.mlpeprocessi(ensemble.innerobj, x, ref y, _params); } /************************************************************************* Relative classification error on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: percent of incorrectly classified cases. Works both for classifier betwork and for regression networks which are used as classifiers. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlperelclserror(mlpensemble ensemble, double[,] xy, int npoints) { return mlpe.mlperelclserror(ensemble.innerobj, xy, npoints, null); } public static double mlperelclserror(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { return mlpe.mlperelclserror(ensemble.innerobj, xy, npoints, _params); } /************************************************************************* Average cross-entropy (in bits per element) on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: CrossEntropy/(NPoints*LN(2)). Zero if ensemble solves regression task. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpeavgce(mlpensemble ensemble, double[,] xy, int npoints) { return mlpe.mlpeavgce(ensemble.innerobj, xy, npoints, null); } public static double mlpeavgce(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { return mlpe.mlpeavgce(ensemble.innerobj, xy, npoints, _params); } /************************************************************************* RMS error on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: root mean square error. Its meaning for regression task is obvious. As for classification task RMS error means error when estimating posterior probabilities. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpermserror(mlpensemble ensemble, double[,] xy, int npoints) { return mlpe.mlpermserror(ensemble.innerobj, xy, npoints, null); } public static double mlpermserror(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { return mlpe.mlpermserror(ensemble.innerobj, xy, npoints, _params); } /************************************************************************* Average error on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: Its meaning for regression task is obvious. As for classification task it means average error when estimating posterior probabilities. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpeavgerror(mlpensemble ensemble, double[,] xy, int npoints) { return mlpe.mlpeavgerror(ensemble.innerobj, xy, npoints, null); } public static double mlpeavgerror(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { return mlpe.mlpeavgerror(ensemble.innerobj, xy, npoints, _params); } /************************************************************************* Average relative error on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: Its meaning for regression task is obvious. As for classification task it means average relative error when estimating posterior probabilities. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpeavgrelerror(mlpensemble ensemble, double[,] xy, int npoints) { return mlpe.mlpeavgrelerror(ensemble.innerobj, xy, npoints, null); } public static double mlpeavgrelerror(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { return mlpe.mlpeavgrelerror(ensemble.innerobj, xy, npoints, _params); } } public partial class alglib { /************************************************************************* Training report: * RelCLSError - fraction of misclassified cases. * AvgCE - acerage cross-entropy * RMSError - root-mean-square error * AvgError - average error * AvgRelError - average relative error * NGrad - number of gradient calculations * NHess - number of Hessian calculations * NCholesky - number of Cholesky decompositions NOTE 1: RelCLSError/AvgCE are zero on regression problems. NOTE 2: on classification problems RMSError/AvgError/AvgRelError contain errors in prediction of posterior probabilities *************************************************************************/ public class mlpreport : alglibobject { // // Public declarations // public double relclserror { get { return _innerobj.relclserror; } set { _innerobj.relclserror = value; } } public double avgce { get { return _innerobj.avgce; } set { _innerobj.avgce = value; } } public double rmserror { get { return _innerobj.rmserror; } set { _innerobj.rmserror = value; } } public double avgerror { get { return _innerobj.avgerror; } set { _innerobj.avgerror = value; } } public double avgrelerror { get { return _innerobj.avgrelerror; } set { _innerobj.avgrelerror = value; } } public int ngrad { get { return _innerobj.ngrad; } set { _innerobj.ngrad = value; } } public int nhess { get { return _innerobj.nhess; } set { _innerobj.nhess = value; } } public int ncholesky { get { return _innerobj.ncholesky; } set { _innerobj.ncholesky = value; } } public mlpreport() { _innerobj = new mlptrain.mlpreport(); } public override alglib.alglibobject make_copy() { return new mlpreport((mlptrain.mlpreport)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private mlptrain.mlpreport _innerobj; public mlptrain.mlpreport innerobj { get { return _innerobj; } } public mlpreport(mlptrain.mlpreport obj) { _innerobj = obj; } } /************************************************************************* Cross-validation estimates of generalization error *************************************************************************/ public class mlpcvreport : alglibobject { // // Public declarations // public double relclserror { get { return _innerobj.relclserror; } set { _innerobj.relclserror = value; } } public double avgce { get { return _innerobj.avgce; } set { _innerobj.avgce = value; } } public double rmserror { get { return _innerobj.rmserror; } set { _innerobj.rmserror = value; } } public double avgerror { get { return _innerobj.avgerror; } set { _innerobj.avgerror = value; } } public double avgrelerror { get { return _innerobj.avgrelerror; } set { _innerobj.avgrelerror = value; } } public mlpcvreport() { _innerobj = new mlptrain.mlpcvreport(); } public override alglib.alglibobject make_copy() { return new mlpcvreport((mlptrain.mlpcvreport)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private mlptrain.mlpcvreport _innerobj; public mlptrain.mlpcvreport innerobj { get { return _innerobj; } } public mlpcvreport(mlptrain.mlpcvreport obj) { _innerobj = obj; } } /************************************************************************* Trainer object for neural network. You should not try to access fields of this object directly - use ALGLIB functions to work with this object. *************************************************************************/ public class mlptrainer : alglibobject { // // Public declarations // public mlptrainer() { _innerobj = new mlptrain.mlptrainer(); } public override alglib.alglibobject make_copy() { return new mlptrainer((mlptrain.mlptrainer)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private mlptrain.mlptrainer _innerobj; public mlptrain.mlptrainer innerobj { get { return _innerobj; } } public mlptrainer(mlptrain.mlptrainer obj) { _innerobj = obj; } } /************************************************************************* Neural network training using modified Levenberg-Marquardt with exact Hessian calculation and regularization. Subroutine trains neural network with restarts from random positions. Algorithm is well suited for small and medium scale problems (hundreds of weights). INPUT PARAMETERS: Network - neural network with initialized geometry XY - training set NPoints - training set size Decay - weight decay constant, >=0.001 Decay term 'Decay*||Weights||^2' is added to error function. If you don't know what Decay to choose, use 0.001. Restarts - number of restarts from random position, >0. If you don't know what Restarts to choose, use 2. OUTPUT PARAMETERS: Network - trained neural network. Info - return code: * -9, if internal matrix inverse subroutine failed * -2, if there is a point with class number outside of [0..NOut-1]. * -1, if wrong parameters specified (NPoints<0, Restarts<1). * 2, if task has been solved. Rep - training report -- ALGLIB -- Copyright 10.03.2009 by Bochkanov Sergey *************************************************************************/ public static void mlptrainlm(multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, out int info, out mlpreport rep) { info = 0; rep = new mlpreport(); mlptrain.mlptrainlm(network.innerobj, xy, npoints, decay, restarts, ref info, rep.innerobj, null); } public static void mlptrainlm(multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, out int info, out mlpreport rep, alglib.xparams _params) { info = 0; rep = new mlpreport(); mlptrain.mlptrainlm(network.innerobj, xy, npoints, decay, restarts, ref info, rep.innerobj, _params); } /************************************************************************* Neural network training using L-BFGS algorithm with regularization. Subroutine trains neural network with restarts from random positions. Algorithm is well suited for problems of any dimensionality (memory requirements and step complexity are linear by weights number). INPUT PARAMETERS: Network - neural network with initialized geometry XY - training set NPoints - training set size Decay - weight decay constant, >=0.001 Decay term 'Decay*||Weights||^2' is added to error function. If you don't know what Decay to choose, use 0.001. Restarts - number of restarts from random position, >0. If you don't know what Restarts to choose, use 2. WStep - stopping criterion. Algorithm stops if step size is less than WStep. Recommended value - 0.01. Zero step size means stopping after MaxIts iterations. MaxIts - stopping criterion. Algorithm stops after MaxIts iterations (NOT gradient calculations). Zero MaxIts means stopping when step is sufficiently small. OUTPUT PARAMETERS: Network - trained neural network. Info - return code: * -8, if both WStep=0 and MaxIts=0 * -2, if there is a point with class number outside of [0..NOut-1]. * -1, if wrong parameters specified (NPoints<0, Restarts<1). * 2, if task has been solved. Rep - training report -- ALGLIB -- Copyright 09.12.2007 by Bochkanov Sergey *************************************************************************/ public static void mlptrainlbfgs(multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, out int info, out mlpreport rep) { info = 0; rep = new mlpreport(); mlptrain.mlptrainlbfgs(network.innerobj, xy, npoints, decay, restarts, wstep, maxits, ref info, rep.innerobj, null); } public static void mlptrainlbfgs(multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, out int info, out mlpreport rep, alglib.xparams _params) { info = 0; rep = new mlpreport(); mlptrain.mlptrainlbfgs(network.innerobj, xy, npoints, decay, restarts, wstep, maxits, ref info, rep.innerobj, _params); } /************************************************************************* Neural network training using early stopping (base algorithm - L-BFGS with regularization). INPUT PARAMETERS: Network - neural network with initialized geometry TrnXY - training set TrnSize - training set size, TrnSize>0 ValXY - validation set ValSize - validation set size, ValSize>0 Decay - weight decay constant, >=0.001 Decay term 'Decay*||Weights||^2' is added to error function. If you don't know what Decay to choose, use 0.001. Restarts - number of restarts, either: * strictly positive number - algorithm make specified number of restarts from random position. * -1, in which case algorithm makes exactly one run from the initial state of the network (no randomization). If you don't know what Restarts to choose, choose one one the following: * -1 (deterministic start) * +1 (one random restart) * +5 (moderate amount of random restarts) OUTPUT PARAMETERS: Network - trained neural network. Info - return code: * -2, if there is a point with class number outside of [0..NOut-1]. * -1, if wrong parameters specified (NPoints<0, Restarts<1, ...). * 2, task has been solved, stopping criterion met - sufficiently small step size. Not expected (we use EARLY stopping) but possible and not an error. * 6, task has been solved, stopping criterion met - increasing of validation set error. Rep - training report NOTE: Algorithm stops if validation set error increases for a long enough or step size is small enought (there are task where validation set may decrease for eternity). In any case solution returned corresponds to the minimum of validation set error. -- ALGLIB -- Copyright 10.03.2009 by Bochkanov Sergey *************************************************************************/ public static void mlptraines(multilayerperceptron network, double[,] trnxy, int trnsize, double[,] valxy, int valsize, double decay, int restarts, out int info, out mlpreport rep) { info = 0; rep = new mlpreport(); mlptrain.mlptraines(network.innerobj, trnxy, trnsize, valxy, valsize, decay, restarts, ref info, rep.innerobj, null); } public static void mlptraines(multilayerperceptron network, double[,] trnxy, int trnsize, double[,] valxy, int valsize, double decay, int restarts, out int info, out mlpreport rep, alglib.xparams _params) { info = 0; rep = new mlpreport(); mlptrain.mlptraines(network.innerobj, trnxy, trnsize, valxy, valsize, decay, restarts, ref info, rep.innerobj, _params); } /************************************************************************* Cross-validation estimate of generalization error. Base algorithm - L-BFGS. INPUT PARAMETERS: Network - neural network with initialized geometry. Network is not changed during cross-validation - it is used only as a representative of its architecture. XY - training set. SSize - training set size Decay - weight decay, same as in MLPTrainLBFGS Restarts - number of restarts, >0. restarts are counted for each partition separately, so total number of restarts will be Restarts*FoldsCount. WStep - stopping criterion, same as in MLPTrainLBFGS MaxIts - stopping criterion, same as in MLPTrainLBFGS FoldsCount - number of folds in k-fold cross-validation, 2<=FoldsCount<=SSize. recommended value: 10. OUTPUT PARAMETERS: Info - return code, same as in MLPTrainLBFGS Rep - report, same as in MLPTrainLM/MLPTrainLBFGS CVRep - generalization error estimates -- ALGLIB -- Copyright 09.12.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpkfoldcvlbfgs(multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, int foldscount, out int info, out mlpreport rep, out mlpcvreport cvrep) { info = 0; rep = new mlpreport(); cvrep = new mlpcvreport(); mlptrain.mlpkfoldcvlbfgs(network.innerobj, xy, npoints, decay, restarts, wstep, maxits, foldscount, ref info, rep.innerobj, cvrep.innerobj, null); } public static void mlpkfoldcvlbfgs(multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, int foldscount, out int info, out mlpreport rep, out mlpcvreport cvrep, alglib.xparams _params) { info = 0; rep = new mlpreport(); cvrep = new mlpcvreport(); mlptrain.mlpkfoldcvlbfgs(network.innerobj, xy, npoints, decay, restarts, wstep, maxits, foldscount, ref info, rep.innerobj, cvrep.innerobj, _params); } /************************************************************************* Cross-validation estimate of generalization error. Base algorithm - Levenberg-Marquardt. INPUT PARAMETERS: Network - neural network with initialized geometry. Network is not changed during cross-validation - it is used only as a representative of its architecture. XY - training set. SSize - training set size Decay - weight decay, same as in MLPTrainLBFGS Restarts - number of restarts, >0. restarts are counted for each partition separately, so total number of restarts will be Restarts*FoldsCount. FoldsCount - number of folds in k-fold cross-validation, 2<=FoldsCount<=SSize. recommended value: 10. OUTPUT PARAMETERS: Info - return code, same as in MLPTrainLBFGS Rep - report, same as in MLPTrainLM/MLPTrainLBFGS CVRep - generalization error estimates -- ALGLIB -- Copyright 09.12.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpkfoldcvlm(multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, int foldscount, out int info, out mlpreport rep, out mlpcvreport cvrep) { info = 0; rep = new mlpreport(); cvrep = new mlpcvreport(); mlptrain.mlpkfoldcvlm(network.innerobj, xy, npoints, decay, restarts, foldscount, ref info, rep.innerobj, cvrep.innerobj, null); } public static void mlpkfoldcvlm(multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, int foldscount, out int info, out mlpreport rep, out mlpcvreport cvrep, alglib.xparams _params) { info = 0; rep = new mlpreport(); cvrep = new mlpcvreport(); mlptrain.mlpkfoldcvlm(network.innerobj, xy, npoints, decay, restarts, foldscount, ref info, rep.innerobj, cvrep.innerobj, _params); } /************************************************************************* This function estimates generalization error using cross-validation on the current dataset with current training settings. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - trainer object Network - neural network. It must have same number of inputs and output/classes as was specified during creation of the trainer object. Network is not changed during cross- validation and is not trained - it is used only as representative of its architecture. I.e., we estimate generalization properties of ARCHITECTURE, not some specific network. NRestarts - number of restarts, >=0: * NRestarts>0 means that for each cross-validation round specified number of random restarts is performed, with best network being chosen after training. * NRestarts=0 is same as NRestarts=1 FoldsCount - number of folds in k-fold cross-validation: * 2<=FoldsCount<=size of dataset * recommended value: 10. * values larger than dataset size will be silently truncated down to dataset size OUTPUT PARAMETERS: Rep - structure which contains cross-validation estimates: * Rep.RelCLSError - fraction of misclassified cases. * Rep.AvgCE - acerage cross-entropy * Rep.RMSError - root-mean-square error * Rep.AvgError - average error * Rep.AvgRelError - average relative error NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(), or subset with only one point was given, zeros are returned as estimates. NOTE: this method performs FoldsCount cross-validation rounds, each one with NRestarts random starts. Thus, FoldsCount*NRestarts networks are trained in total. NOTE: Rep.RelCLSError/Rep.AvgCE are zero on regression problems. NOTE: on classification problems Rep.RMSError/Rep.AvgError/Rep.AvgRelError contain errors in prediction of posterior probabilities. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpkfoldcv(mlptrainer s, multilayerperceptron network, int nrestarts, int foldscount, out mlpreport rep) { rep = new mlpreport(); mlptrain.mlpkfoldcv(s.innerobj, network.innerobj, nrestarts, foldscount, rep.innerobj, null); } public static void mlpkfoldcv(mlptrainer s, multilayerperceptron network, int nrestarts, int foldscount, out mlpreport rep, alglib.xparams _params) { rep = new mlpreport(); mlptrain.mlpkfoldcv(s.innerobj, network.innerobj, nrestarts, foldscount, rep.innerobj, _params); } /************************************************************************* Creation of the network trainer object for regression networks INPUT PARAMETERS: NIn - number of inputs, NIn>=1 NOut - number of outputs, NOut>=1 OUTPUT PARAMETERS: S - neural network trainer object. This structure can be used to train any regression network with NIn inputs and NOut outputs. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatetrainer(int nin, int nout, out mlptrainer s) { s = new mlptrainer(); mlptrain.mlpcreatetrainer(nin, nout, s.innerobj, null); } public static void mlpcreatetrainer(int nin, int nout, out mlptrainer s, alglib.xparams _params) { s = new mlptrainer(); mlptrain.mlpcreatetrainer(nin, nout, s.innerobj, _params); } /************************************************************************* Creation of the network trainer object for classification networks INPUT PARAMETERS: NIn - number of inputs, NIn>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: S - neural network trainer object. This structure can be used to train any classification network with NIn inputs and NOut outputs. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatetrainercls(int nin, int nclasses, out mlptrainer s) { s = new mlptrainer(); mlptrain.mlpcreatetrainercls(nin, nclasses, s.innerobj, null); } public static void mlpcreatetrainercls(int nin, int nclasses, out mlptrainer s, alglib.xparams _params) { s = new mlptrainer(); mlptrain.mlpcreatetrainercls(nin, nclasses, s.innerobj, _params); } /************************************************************************* This function sets "current dataset" of the trainer object to one passed by user. INPUT PARAMETERS: S - trainer object XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. NPoints - points count, >=0. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following datasetformat is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetdataset(mlptrainer s, double[,] xy, int npoints) { mlptrain.mlpsetdataset(s.innerobj, xy, npoints, null); } public static void mlpsetdataset(mlptrainer s, double[,] xy, int npoints, alglib.xparams _params) { mlptrain.mlpsetdataset(s.innerobj, xy, npoints, _params); } /************************************************************************* This function sets "current dataset" of the trainer object to one passed by user (sparse matrix is used to store dataset). INPUT PARAMETERS: S - trainer object XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Any sparse storage format can be used: Hash-table, CRS... NPoints - points count, >=0 DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following datasetformat is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetsparsedataset(mlptrainer s, sparsematrix xy, int npoints) { mlptrain.mlpsetsparsedataset(s.innerobj, xy.innerobj, npoints, null); } public static void mlpsetsparsedataset(mlptrainer s, sparsematrix xy, int npoints, alglib.xparams _params) { mlptrain.mlpsetsparsedataset(s.innerobj, xy.innerobj, npoints, _params); } /************************************************************************* This function sets weight decay coefficient which is used for training. INPUT PARAMETERS: S - trainer object Decay - weight decay coefficient, >=0. Weight decay term 'Decay*||Weights||^2' is added to error function. If you don't know what Decay to choose, use 1.0E-3. Weight decay can be set to zero, in this case network is trained without weight decay. NOTE: by default network uses some small nonzero value for weight decay. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetdecay(mlptrainer s, double decay) { mlptrain.mlpsetdecay(s.innerobj, decay, null); } public static void mlpsetdecay(mlptrainer s, double decay, alglib.xparams _params) { mlptrain.mlpsetdecay(s.innerobj, decay, _params); } /************************************************************************* This function sets stopping criteria for the optimizer. INPUT PARAMETERS: S - trainer object WStep - stopping criterion. Algorithm stops if step size is less than WStep. Recommended value - 0.01. Zero step size means stopping after MaxIts iterations. WStep>=0. MaxIts - stopping criterion. Algorithm stops after MaxIts epochs (full passes over entire dataset). Zero MaxIts means stopping when step is sufficiently small. MaxIts>=0. NOTE: by default, WStep=0.005 and MaxIts=0 are used. These values are also used when MLPSetCond() is called with WStep=0 and MaxIts=0. NOTE: these stopping criteria are used for all kinds of neural training - from "conventional" networks to early stopping ensembles. When used for "conventional" networks, they are used as the only stopping criteria. When combined with early stopping, they used as ADDITIONAL stopping criteria which can terminate early stopping algorithm. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetcond(mlptrainer s, double wstep, int maxits) { mlptrain.mlpsetcond(s.innerobj, wstep, maxits, null); } public static void mlpsetcond(mlptrainer s, double wstep, int maxits, alglib.xparams _params) { mlptrain.mlpsetcond(s.innerobj, wstep, maxits, _params); } /************************************************************************* This function sets training algorithm: batch training using L-BFGS will be used. This algorithm: * the most robust for small-scale problems, but may be too slow for large scale ones. * perfoms full pass through the dataset before performing step * uses conditions specified by MLPSetCond() for stopping * is default one used by trainer object INPUT PARAMETERS: S - trainer object -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetalgobatch(mlptrainer s) { mlptrain.mlpsetalgobatch(s.innerobj, null); } public static void mlpsetalgobatch(mlptrainer s, alglib.xparams _params) { mlptrain.mlpsetalgobatch(s.innerobj, _params); } /************************************************************************* This function trains neural network passed to this function, using current dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset()) and current training settings. Training from NRestarts random starting positions is performed, best network is chosen. Training is performed using current training algorithm. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - trainer object Network - neural network. It must have same number of inputs and output/classes as was specified during creation of the trainer object. NRestarts - number of restarts, >=0: * NRestarts>0 means that specified number of random restarts are performed, best network is chosen after training * NRestarts=0 means that current state of the network is used for training. OUTPUT PARAMETERS: Network - trained network NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(), network is filled by zero values. Same behavior for functions MLPStartTraining and MLPContinueTraining. NOTE: this method uses sum-of-squares error function for training. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlptrainnetwork(mlptrainer s, multilayerperceptron network, int nrestarts, out mlpreport rep) { rep = new mlpreport(); mlptrain.mlptrainnetwork(s.innerobj, network.innerobj, nrestarts, rep.innerobj, null); } public static void mlptrainnetwork(mlptrainer s, multilayerperceptron network, int nrestarts, out mlpreport rep, alglib.xparams _params) { rep = new mlpreport(); mlptrain.mlptrainnetwork(s.innerobj, network.innerobj, nrestarts, rep.innerobj, _params); } /************************************************************************* IMPORTANT: this is an "expert" version of the MLPTrain() function. We do not recommend you to use it unless you are pretty sure that you need ability to monitor training progress. This function performs step-by-step training of the neural network. Here "step-by-step" means that training starts with MLPStartTraining() call, and then user subsequently calls MLPContinueTraining() to perform one more iteration of the training. After call to this function trainer object remembers network and is ready to train it. However, no training is performed until first call to MLPContinueTraining() function. Subsequent calls to MLPContinueTraining() will advance training progress one iteration further. EXAMPLE: > > ...initialize network and trainer object.... > > MLPStartTraining(Trainer, Network, True) > while MLPContinueTraining(Trainer, Network) do > ...visualize training progress... > INPUT PARAMETERS: S - trainer object Network - neural network. It must have same number of inputs and output/classes as was specified during creation of the trainer object. RandomStart - randomize network before training or not: * True means that network is randomized and its initial state (one which was passed to the trainer object) is lost. * False means that training is started from the current state of the network OUTPUT PARAMETERS: Network - neural network which is ready to training (weights are initialized, preprocessor is initialized using current training set) NOTE: this method uses sum-of-squares error function for training. NOTE: it is expected that trainer object settings are NOT changed during step-by-step training, i.e. no one changes stopping criteria or training set during training. It is possible and there is no defense against such actions, but algorithm behavior in such cases is undefined and can be unpredictable. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpstarttraining(mlptrainer s, multilayerperceptron network, bool randomstart) { mlptrain.mlpstarttraining(s.innerobj, network.innerobj, randomstart, null); } public static void mlpstarttraining(mlptrainer s, multilayerperceptron network, bool randomstart, alglib.xparams _params) { mlptrain.mlpstarttraining(s.innerobj, network.innerobj, randomstart, _params); } /************************************************************************* IMPORTANT: this is an "expert" version of the MLPTrain() function. We do not recommend you to use it unless you are pretty sure that you need ability to monitor training progress. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. This function performs step-by-step training of the neural network. Here "step-by-step" means that training starts with MLPStartTraining() call, and then user subsequently calls MLPContinueTraining() to perform one more iteration of the training. This function performs one more iteration of the training and returns either True (training continues) or False (training stopped). In case True was returned, Network weights are updated according to the current state of the optimization progress. In case False was returned, no additional updates is performed (previous update of the network weights moved us to the final point, and no additional updates is needed). EXAMPLE: > > [initialize network and trainer object] > > MLPStartTraining(Trainer, Network, True) > while MLPContinueTraining(Trainer, Network) do > [visualize training progress] > INPUT PARAMETERS: S - trainer object Network - neural network structure, which is used to store current state of the training process. OUTPUT PARAMETERS: Network - weights of the neural network are rewritten by the current approximation. NOTE: this method uses sum-of-squares error function for training. NOTE: it is expected that trainer object settings are NOT changed during step-by-step training, i.e. no one changes stopping criteria or training set during training. It is possible and there is no defense against such actions, but algorithm behavior in such cases is undefined and can be unpredictable. NOTE: It is expected that Network is the same one which was passed to MLPStartTraining() function. However, THIS function checks only following: * that number of network inputs is consistent with trainer object settings * that number of network outputs/classes is consistent with trainer object settings * that number of network weights is the same as number of weights in the network passed to MLPStartTraining() function Exception is thrown when these conditions are violated. It is also expected that you do not change state of the network on your own - the only party who has right to change network during its training is a trainer object. Any attempt to interfere with trainer may lead to unpredictable results. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static bool mlpcontinuetraining(mlptrainer s, multilayerperceptron network) { return mlptrain.mlpcontinuetraining(s.innerobj, network.innerobj, null); } public static bool mlpcontinuetraining(mlptrainer s, multilayerperceptron network, alglib.xparams _params) { return mlptrain.mlpcontinuetraining(s.innerobj, network.innerobj, _params); } /************************************************************************* Training neural networks ensemble using bootstrap aggregating (bagging). Modified Levenberg-Marquardt algorithm is used as base training method. INPUT PARAMETERS: Ensemble - model with initialized geometry XY - training set NPoints - training set size Decay - weight decay coefficient, >=0.001 Restarts - restarts, >0. OUTPUT PARAMETERS: Ensemble - trained model Info - return code: * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, Restarts<1). * 2, if task has been solved. Rep - training report. OOBErrors - out-of-bag generalization error estimate -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpebagginglm(mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, out int info, out mlpreport rep, out mlpcvreport ooberrors) { info = 0; rep = new mlpreport(); ooberrors = new mlpcvreport(); mlptrain.mlpebagginglm(ensemble.innerobj, xy, npoints, decay, restarts, ref info, rep.innerobj, ooberrors.innerobj, null); } public static void mlpebagginglm(mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, out int info, out mlpreport rep, out mlpcvreport ooberrors, alglib.xparams _params) { info = 0; rep = new mlpreport(); ooberrors = new mlpcvreport(); mlptrain.mlpebagginglm(ensemble.innerobj, xy, npoints, decay, restarts, ref info, rep.innerobj, ooberrors.innerobj, _params); } /************************************************************************* Training neural networks ensemble using bootstrap aggregating (bagging). L-BFGS algorithm is used as base training method. INPUT PARAMETERS: Ensemble - model with initialized geometry XY - training set NPoints - training set size Decay - weight decay coefficient, >=0.001 Restarts - restarts, >0. WStep - stopping criterion, same as in MLPTrainLBFGS MaxIts - stopping criterion, same as in MLPTrainLBFGS OUTPUT PARAMETERS: Ensemble - trained model Info - return code: * -8, if both WStep=0 and MaxIts=0 * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, Restarts<1). * 2, if task has been solved. Rep - training report. OOBErrors - out-of-bag generalization error estimate -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpebagginglbfgs(mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, out int info, out mlpreport rep, out mlpcvreport ooberrors) { info = 0; rep = new mlpreport(); ooberrors = new mlpcvreport(); mlptrain.mlpebagginglbfgs(ensemble.innerobj, xy, npoints, decay, restarts, wstep, maxits, ref info, rep.innerobj, ooberrors.innerobj, null); } public static void mlpebagginglbfgs(mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, out int info, out mlpreport rep, out mlpcvreport ooberrors, alglib.xparams _params) { info = 0; rep = new mlpreport(); ooberrors = new mlpcvreport(); mlptrain.mlpebagginglbfgs(ensemble.innerobj, xy, npoints, decay, restarts, wstep, maxits, ref info, rep.innerobj, ooberrors.innerobj, _params); } /************************************************************************* Training neural networks ensemble using early stopping. INPUT PARAMETERS: Ensemble - model with initialized geometry XY - training set NPoints - training set size Decay - weight decay coefficient, >=0.001 Restarts - restarts, >0. OUTPUT PARAMETERS: Ensemble - trained model Info - return code: * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, Restarts<1). * 6, if task has been solved. Rep - training report. OOBErrors - out-of-bag generalization error estimate -- ALGLIB -- Copyright 10.03.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpetraines(mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, out int info, out mlpreport rep) { info = 0; rep = new mlpreport(); mlptrain.mlpetraines(ensemble.innerobj, xy, npoints, decay, restarts, ref info, rep.innerobj, null); } public static void mlpetraines(mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, out int info, out mlpreport rep, alglib.xparams _params) { info = 0; rep = new mlpreport(); mlptrain.mlpetraines(ensemble.innerobj, xy, npoints, decay, restarts, ref info, rep.innerobj, _params); } /************************************************************************* This function trains neural network ensemble passed to this function using current dataset and early stopping training algorithm. Each early stopping round performs NRestarts random restarts (thus, EnsembleSize*NRestarts training rounds is performed in total). ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - trainer object; Ensemble - neural network ensemble. It must have same number of inputs and outputs/classes as was specified during creation of the trainer object. NRestarts - number of restarts, >=0: * NRestarts>0 means that specified number of random restarts are performed during each ES round; * NRestarts=0 is silently replaced by 1. OUTPUT PARAMETERS: Ensemble - trained ensemble; Rep - it contains all type of errors. NOTE: this training method uses BOTH early stopping and weight decay! So, you should select weight decay before starting training just as you select it before training "conventional" networks. NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(), or single-point dataset was passed, ensemble is filled by zero values. NOTE: this method uses sum-of-squares error function for training. -- ALGLIB -- Copyright 22.08.2012 by Bochkanov Sergey *************************************************************************/ public static void mlptrainensemblees(mlptrainer s, mlpensemble ensemble, int nrestarts, out mlpreport rep) { rep = new mlpreport(); mlptrain.mlptrainensemblees(s.innerobj, ensemble.innerobj, nrestarts, rep.innerobj, null); } public static void mlptrainensemblees(mlptrainer s, mlpensemble ensemble, int nrestarts, out mlpreport rep, alglib.xparams _params) { rep = new mlpreport(); mlptrain.mlptrainensemblees(s.innerobj, ensemble.innerobj, nrestarts, rep.innerobj, _params); } } public partial class alglib { /************************************************************************* This structure is a clusterization engine. You should not try to access its fields directly. Use ALGLIB functions in order to work with this object. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public class clusterizerstate : alglibobject { // // Public declarations // public clusterizerstate() { _innerobj = new clustering.clusterizerstate(); } public override alglib.alglibobject make_copy() { return new clusterizerstate((clustering.clusterizerstate)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private clustering.clusterizerstate _innerobj; public clustering.clusterizerstate innerobj { get { return _innerobj; } } public clusterizerstate(clustering.clusterizerstate obj) { _innerobj = obj; } } /************************************************************************* This structure is used to store results of the agglomerative hierarchical clustering (AHC). Following information is returned: * TerminationType - completion code: * 1 for successful completion of algorithm * -5 inappropriate combination of clustering algorithm and distance function was used. As for now, it is possible only when Ward's method is called for dataset with non-Euclidean distance function. In case negative completion code is returned, other fields of report structure are invalid and should not be used. * NPoints contains number of points in the original dataset * Z contains information about merges performed (see below). Z contains indexes from the original (unsorted) dataset and it can be used when you need to know what points were merged. However, it is not convenient when you want to build a dendrograd (see below). * if you want to build dendrogram, you can use Z, but it is not good option, because Z contains indexes from unsorted dataset. Dendrogram built from such dataset is likely to have intersections. So, you have to reorder you points before building dendrogram. Permutation which reorders point is returned in P. Another representation of merges, which is more convenient for dendorgram construction, is returned in PM. * more information on format of Z, P and PM can be found below and in the examples from ALGLIB Reference Manual. FORMAL DESCRIPTION OF FIELDS: NPoints number of points Z array[NPoints-1,2], contains indexes of clusters linked in pairs to form clustering tree. I-th row corresponds to I-th merge: * Z[I,0] - index of the first cluster to merge * Z[I,1] - index of the second cluster to merge * Z[I,0]=0 NFeatures number of variables, >=1 TerminationType completion code: * -5 if distance type is anything different from Euclidean metric * -3 for degenerate dataset: a) less than K distinct points, b) K=0 for non-empty dataset. * +1 for successful completion K number of clusters C array[K,NFeatures], rows of the array store centers CIdx array[NPoints], which contains cluster indexes IterationsCount actual number of iterations performed by clusterizer. If algorithm performed more than one random restart, total number of iterations is returned. Energy merit function, "energy", sum of squared deviations from cluster centers -- ALGLIB -- Copyright 27.11.2012 by Bochkanov Sergey *************************************************************************/ public class kmeansreport : alglibobject { // // Public declarations // public int npoints { get { return _innerobj.npoints; } set { _innerobj.npoints = value; } } public int nfeatures { get { return _innerobj.nfeatures; } set { _innerobj.nfeatures = value; } } public int terminationtype { get { return _innerobj.terminationtype; } set { _innerobj.terminationtype = value; } } public int iterationscount { get { return _innerobj.iterationscount; } set { _innerobj.iterationscount = value; } } public double energy { get { return _innerobj.energy; } set { _innerobj.energy = value; } } public int k { get { return _innerobj.k; } set { _innerobj.k = value; } } public double[,] c { get { return _innerobj.c; } set { _innerobj.c = value; } } public int[] cidx { get { return _innerobj.cidx; } set { _innerobj.cidx = value; } } public kmeansreport() { _innerobj = new clustering.kmeansreport(); } public override alglib.alglibobject make_copy() { return new kmeansreport((clustering.kmeansreport)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private clustering.kmeansreport _innerobj; public clustering.kmeansreport innerobj { get { return _innerobj; } } public kmeansreport(clustering.kmeansreport obj) { _innerobj = obj; } } /************************************************************************* This function initializes clusterizer object. Newly initialized object is empty, i.e. it does not contain dataset. You should use it as follows: 1. creation 2. dataset is added with ClusterizerSetPoints() 3. additional parameters are set 3. clusterization is performed with one of the clustering functions -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizercreate(out clusterizerstate s) { s = new clusterizerstate(); clustering.clusterizercreate(s.innerobj, null); } public static void clusterizercreate(out clusterizerstate s, alglib.xparams _params) { s = new clusterizerstate(); clustering.clusterizercreate(s.innerobj, _params); } /************************************************************************* This function adds dataset to the clusterizer structure. This function overrides all previous calls of ClusterizerSetPoints() or ClusterizerSetDistances(). INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() XY - array[NPoints,NFeatures], dataset NPoints - number of points, >=0 NFeatures- number of features, >=1 DistType- distance function: * 0 Chebyshev distance (L-inf norm) * 1 city block distance (L1 norm) * 2 Euclidean distance (L2 norm), non-squared * 10 Pearson correlation: dist(a,b) = 1-corr(a,b) * 11 Absolute Pearson correlation: dist(a,b) = 1-|corr(a,b)| * 12 Uncentered Pearson correlation (cosine of the angle): dist(a,b) = a'*b/(|a|*|b|) * 13 Absolute uncentered Pearson correlation dist(a,b) = |a'*b|/(|a|*|b|) * 20 Spearman rank correlation: dist(a,b) = 1-rankcorr(a,b) * 21 Absolute Spearman rank correlation dist(a,b) = 1-|rankcorr(a,b)| NOTE 1: different distance functions have different performance penalty: * Euclidean or Pearson correlation distances are the fastest ones * Spearman correlation distance function is a bit slower * city block and Chebyshev distances are order of magnitude slower The reason behing difference in performance is that correlation-based distance functions are computed using optimized linear algebra kernels, while Chebyshev and city block distance functions are computed using simple nested loops with two branches at each iteration. NOTE 2: different clustering algorithms have different limitations: * agglomerative hierarchical clustering algorithms may be used with any kind of distance metric * k-means++ clustering algorithm may be used only with Euclidean distance function Thus, list of specific clustering algorithms you may use depends on distance function you specify when you set your dataset. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetpoints(clusterizerstate s, double[,] xy, int npoints, int nfeatures, int disttype) { clustering.clusterizersetpoints(s.innerobj, xy, npoints, nfeatures, disttype, null); } public static void clusterizersetpoints(clusterizerstate s, double[,] xy, int npoints, int nfeatures, int disttype, alglib.xparams _params) { clustering.clusterizersetpoints(s.innerobj, xy, npoints, nfeatures, disttype, _params); } public static void clusterizersetpoints(clusterizerstate s, double[,] xy, int disttype) { int npoints; int nfeatures; npoints = ap.rows(xy); nfeatures = ap.cols(xy); clustering.clusterizersetpoints(s.innerobj, xy, npoints, nfeatures, disttype, null); return; } public static void clusterizersetpoints(clusterizerstate s, double[,] xy, int disttype, alglib.xparams _params) { int npoints; int nfeatures; npoints = ap.rows(xy); nfeatures = ap.cols(xy); clustering.clusterizersetpoints(s.innerobj, xy, npoints, nfeatures, disttype, _params); return; } /************************************************************************* This function adds dataset given by distance matrix to the clusterizer structure. It is important that dataset is not given explicitly - only distance matrix is given. This function overrides all previous calls of ClusterizerSetPoints() or ClusterizerSetDistances(). INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() D - array[NPoints,NPoints], distance matrix given by its upper or lower triangle (main diagonal is ignored because its entries are expected to be zero). NPoints - number of points IsUpper - whether upper or lower triangle of D is given. NOTE 1: different clustering algorithms have different limitations: * agglomerative hierarchical clustering algorithms may be used with any kind of distance metric, including one which is given by distance matrix * k-means++ clustering algorithm may be used only with Euclidean distance function and explicitly given points - it can not be used with dataset given by distance matrix Thus, if you call this function, you will be unable to use k-means clustering algorithm to process your problem. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetdistances(clusterizerstate s, double[,] d, int npoints, bool isupper) { clustering.clusterizersetdistances(s.innerobj, d, npoints, isupper, null); } public static void clusterizersetdistances(clusterizerstate s, double[,] d, int npoints, bool isupper, alglib.xparams _params) { clustering.clusterizersetdistances(s.innerobj, d, npoints, isupper, _params); } public static void clusterizersetdistances(clusterizerstate s, double[,] d, bool isupper) { int npoints; if( (ap.rows(d)!=ap.cols(d))) throw new alglibexception("Error while calling 'clusterizersetdistances': looks like one of arguments has wrong size"); npoints = ap.rows(d); clustering.clusterizersetdistances(s.innerobj, d, npoints, isupper, null); return; } public static void clusterizersetdistances(clusterizerstate s, double[,] d, bool isupper, alglib.xparams _params) { int npoints; if( (ap.rows(d)!=ap.cols(d))) throw new alglibexception("Error while calling 'clusterizersetdistances': looks like one of arguments has wrong size"); npoints = ap.rows(d); clustering.clusterizersetdistances(s.innerobj, d, npoints, isupper, _params); return; } /************************************************************************* This function sets agglomerative hierarchical clustering algorithm INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() Algo - algorithm type: * 0 complete linkage (default algorithm) * 1 single linkage * 2 unweighted average linkage * 3 weighted average linkage * 4 Ward's method NOTE: Ward's method works correctly only with Euclidean distance, that's why algorithm will return negative termination code (failure) for any other distance type. It is possible, however, to use this method with user-supplied distance matrix. It is your responsibility to pass one which was calculated with Euclidean distance function. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetahcalgo(clusterizerstate s, int algo) { clustering.clusterizersetahcalgo(s.innerobj, algo, null); } public static void clusterizersetahcalgo(clusterizerstate s, int algo, alglib.xparams _params) { clustering.clusterizersetahcalgo(s.innerobj, algo, _params); } /************************************************************************* This function sets k-means properties: number of restarts and maximum number of iterations per one run. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() Restarts- restarts count, >=1. k-means++ algorithm performs several restarts and chooses best set of centers (one with minimum squared distance). MaxIts - maximum number of k-means iterations performed during one run. >=0, zero value means that algorithm performs unlimited number of iterations. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetkmeanslimits(clusterizerstate s, int restarts, int maxits) { clustering.clusterizersetkmeanslimits(s.innerobj, restarts, maxits, null); } public static void clusterizersetkmeanslimits(clusterizerstate s, int restarts, int maxits, alglib.xparams _params) { clustering.clusterizersetkmeanslimits(s.innerobj, restarts, maxits, _params); } /************************************************************************* This function sets k-means initialization algorithm. Several different algorithms can be chosen, including k-means++. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() InitAlgo- initialization algorithm: * 0 automatic selection ( different versions of ALGLIB may select different algorithms) * 1 random initialization * 2 k-means++ initialization (best quality of initial centers, but long non-parallelizable initialization phase with bad cache locality) * 3 "fast-greedy" algorithm with efficient, easy to parallelize initialization. Quality of initial centers is somewhat worse than that of k-means++. This algorithm is a default one in the current version of ALGLIB. *-1 "debug" algorithm which always selects first K rows of dataset; this algorithm is used for debug purposes only. Do not use it in the industrial code! -- ALGLIB -- Copyright 21.01.2015 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetkmeansinit(clusterizerstate s, int initalgo) { clustering.clusterizersetkmeansinit(s.innerobj, initalgo, null); } public static void clusterizersetkmeansinit(clusterizerstate s, int initalgo, alglib.xparams _params) { clustering.clusterizersetkmeansinit(s.innerobj, initalgo, _params); } /************************************************************************* This function sets seed which is used to initialize internal RNG. By default, deterministic seed is used - same for each run of clusterizer. If you specify non-deterministic seed value, then some algorithms which depend on random initialization (in current version: k-means) may return slightly different results after each run. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() Seed - seed: * positive values = use deterministic seed for each run of algorithms which depend on random initialization * zero or negative values = use non-deterministic seed -- ALGLIB -- Copyright 08.06.2017 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetseed(clusterizerstate s, int seed) { clustering.clusterizersetseed(s.innerobj, seed, null); } public static void clusterizersetseed(clusterizerstate s, int seed, alglib.xparams _params) { clustering.clusterizersetseed(s.innerobj, seed, _params); } /************************************************************************* This function performs agglomerative hierarchical clustering ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. NOTE: Agglomerative hierarchical clustering algorithm has two phases: distance matrix calculation and clustering itself. Only first phase (distance matrix calculation) is accelerated by Intel MKL and multithreading. Thus, acceleration is significant only for medium or high-dimensional problems. Although activating multithreading gives some speedup over single- threaded execution, you should not expect nearly-linear scaling with respect to cores count. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() OUTPUT PARAMETERS: Rep - clustering results; see description of AHCReport structure for more information. NOTE 1: hierarchical clustering algorithms require large amounts of memory. In particular, this implementation needs sizeof(double)*NPoints^2 bytes, which are used to store distance matrix. In case we work with user-supplied matrix, this amount is multiplied by 2 (we have to store original matrix and to work with its copy). For example, problem with 10000 points would require 800M of RAM, even when working in a 1-dimensional space. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizerrunahc(clusterizerstate s, out ahcreport rep) { rep = new ahcreport(); clustering.clusterizerrunahc(s.innerobj, rep.innerobj, null); } public static void clusterizerrunahc(clusterizerstate s, out ahcreport rep, alglib.xparams _params) { rep = new ahcreport(); clustering.clusterizerrunahc(s.innerobj, rep.innerobj, _params); } /************************************************************************* This function performs clustering by k-means++ algorithm. You may change algorithm properties by calling: * ClusterizerSetKMeansLimits() to change number of restarts or iterations * ClusterizerSetKMeansInit() to change initialization algorithm By default, one restart and unlimited number of iterations are used. Initialization algorithm is chosen automatically. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. NOTE: k-means clustering algorithm has two phases: selection of initial centers and clustering itself. ALGLIB parallelizes both phases. Parallel version is optimized for the following scenario: medium or high-dimensional problem (8 or more dimensions) with large number of points and clusters. However, some speed-up can be obtained even when assumptions above are violated. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() K - number of clusters, K>=0. K can be zero only when algorithm is called for empty dataset, in this case completion code is set to success (+1). If K=0 and dataset size is non-zero, we can not meaningfully assign points to some center (there are no centers because K=0) and return -3 as completion code (failure). OUTPUT PARAMETERS: Rep - clustering results; see description of KMeansReport structure for more information. NOTE 1: k-means clustering can be performed only for datasets with Euclidean distance function. Algorithm will return negative completion code in Rep.TerminationType in case dataset was added to clusterizer with DistType other than Euclidean (or dataset was specified by distance matrix instead of explicitly given points). NOTE 2: by default, k-means uses non-deterministic seed to initialize RNG which is used to select initial centers. As result, each run of algorithm may return different values. If you need deterministic behavior, use ClusterizerSetSeed() function. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizerrunkmeans(clusterizerstate s, int k, out kmeansreport rep) { rep = new kmeansreport(); clustering.clusterizerrunkmeans(s.innerobj, k, rep.innerobj, null); } public static void clusterizerrunkmeans(clusterizerstate s, int k, out kmeansreport rep, alglib.xparams _params) { rep = new kmeansreport(); clustering.clusterizerrunkmeans(s.innerobj, k, rep.innerobj, _params); } /************************************************************************* This function returns distance matrix for dataset ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: XY - array[NPoints,NFeatures], dataset NPoints - number of points, >=0 NFeatures- number of features, >=1 DistType- distance function: * 0 Chebyshev distance (L-inf norm) * 1 city block distance (L1 norm) * 2 Euclidean distance (L2 norm, non-squared) * 10 Pearson correlation: dist(a,b) = 1-corr(a,b) * 11 Absolute Pearson correlation: dist(a,b) = 1-|corr(a,b)| * 12 Uncentered Pearson correlation (cosine of the angle): dist(a,b) = a'*b/(|a|*|b|) * 13 Absolute uncentered Pearson correlation dist(a,b) = |a'*b|/(|a|*|b|) * 20 Spearman rank correlation: dist(a,b) = 1-rankcorr(a,b) * 21 Absolute Spearman rank correlation dist(a,b) = 1-|rankcorr(a,b)| OUTPUT PARAMETERS: D - array[NPoints,NPoints], distance matrix (full matrix is returned, with lower and upper triangles) NOTE: different distance functions have different performance penalty: * Euclidean or Pearson correlation distances are the fastest ones * Spearman correlation distance function is a bit slower * city block and Chebyshev distances are order of magnitude slower The reason behing difference in performance is that correlation-based distance functions are computed using optimized linear algebra kernels, while Chebyshev and city block distance functions are computed using simple nested loops with two branches at each iteration. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizergetdistances(double[,] xy, int npoints, int nfeatures, int disttype, out double[,] d) { d = new double[0,0]; clustering.clusterizergetdistances(xy, npoints, nfeatures, disttype, ref d, null); } public static void clusterizergetdistances(double[,] xy, int npoints, int nfeatures, int disttype, out double[,] d, alglib.xparams _params) { d = new double[0,0]; clustering.clusterizergetdistances(xy, npoints, nfeatures, disttype, ref d, _params); } /************************************************************************* This function takes as input clusterization report Rep, desired clusters count K, and builds top K clusters from hierarchical clusterization tree. It returns assignment of points to clusters (array of cluster indexes). INPUT PARAMETERS: Rep - report from ClusterizerRunAHC() performed on XY K - desired number of clusters, 1<=K<=NPoints. K can be zero only when NPoints=0. OUTPUT PARAMETERS: CIdx - array[NPoints], I-th element contains cluster index (from 0 to K-1) for I-th point of the dataset. CZ - array[K]. This array allows to convert cluster indexes returned by this function to indexes used by Rep.Z. J-th cluster returned by this function corresponds to CZ[J]-th cluster stored in Rep.Z/PZ/PM. It is guaranteed that CZ[I]=0 OUTPUT PARAMETERS: K - number of clusters, 1<=K<=NPoints CIdx - array[NPoints], I-th element contains cluster index (from 0 to K-1) for I-th point of the dataset. CZ - array[K]. This array allows to convert cluster indexes returned by this function to indexes used by Rep.Z. J-th cluster returned by this function corresponds to CZ[J]-th cluster stored in Rep.Z/PZ/PM. It is guaranteed that CZ[I]=1 NVars - number of independent variables, NVars>=1 NClasses - indicates type of the problem being solved: * NClasses>=2 means that classification problem is solved (last column of the dataset stores class number) * NClasses=1 means that regression problem is solved (last column of the dataset stores variable value) OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetdataset(decisionforestbuilder s, double[,] xy, int npoints, int nvars, int nclasses) { dforest.dfbuildersetdataset(s.innerobj, xy, npoints, nvars, nclasses, null); } public static void dfbuildersetdataset(decisionforestbuilder s, double[,] xy, int npoints, int nvars, int nclasses, alglib.xparams _params) { dforest.dfbuildersetdataset(s.innerobj, xy, npoints, nvars, nclasses, _params); } /************************************************************************* This function sets number of variables (in [1,NVars] range) used by decision forest construction algorithm. The default option is to use roughly sqrt(NVars) variables. INPUT PARAMETERS: S - decision forest builder object RndVars - number of randomly selected variables; values outside of [1,NVars] range are silently clipped. OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrndvars(decisionforestbuilder s, int rndvars) { dforest.dfbuildersetrndvars(s.innerobj, rndvars, null); } public static void dfbuildersetrndvars(decisionforestbuilder s, int rndvars, alglib.xparams _params) { dforest.dfbuildersetrndvars(s.innerobj, rndvars, _params); } /************************************************************************* This function sets number of variables used by decision forest construction algorithm as a fraction of total variable count (0,1) range. The default option is to use roughly sqrt(NVars) variables. INPUT PARAMETERS: S - decision forest builder object F - round(NVars*F) variables are selected OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrndvarsratio(decisionforestbuilder s, double f) { dforest.dfbuildersetrndvarsratio(s.innerobj, f, null); } public static void dfbuildersetrndvarsratio(decisionforestbuilder s, double f, alglib.xparams _params) { dforest.dfbuildersetrndvarsratio(s.innerobj, f, _params); } /************************************************************************* This function tells decision forest builder to automatically choose number of variables used by decision forest construction algorithm. Roughly sqrt(NVars) variables will be used. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrndvarsauto(decisionforestbuilder s) { dforest.dfbuildersetrndvarsauto(s.innerobj, null); } public static void dfbuildersetrndvarsauto(decisionforestbuilder s, alglib.xparams _params) { dforest.dfbuildersetrndvarsauto(s.innerobj, _params); } /************************************************************************* This function sets size of dataset subsample generated the decision forest construction algorithm. Size is specified as a fraction of total dataset size. The default option is to use 50% of the dataset for training, 50% for the OOB estimates. You can decrease fraction F down to 10%, 1% or even below in order to reduce overfitting. INPUT PARAMETERS: S - decision forest builder object F - fraction of the dataset to use, in (0,1] range. Values outside of this range will be silently clipped. At least one element is always selected for the training set. OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetsubsampleratio(decisionforestbuilder s, double f) { dforest.dfbuildersetsubsampleratio(s.innerobj, f, null); } public static void dfbuildersetsubsampleratio(decisionforestbuilder s, double f, alglib.xparams _params) { dforest.dfbuildersetsubsampleratio(s.innerobj, f, _params); } /************************************************************************* This function sets seed used by internal RNG for random subsampling and random selection of variable subsets. By default random seed is used, i.e. every time you build decision forest, we seed generator with new value obtained from system-wide RNG. Thus, decision forest builder returns non-deterministic results. You can change such behavior by specyfing fixed positive seed value. INPUT PARAMETERS: S - decision forest builder object SeedVal - seed value: * positive values are used for seeding RNG with fixed seed, i.e. subsequent runs on same data will return same decision forests * non-positive seed means that random seed is used for every run of builder, i.e. subsequent runs on same datasets will return slightly different decision forests OUTPUT PARAMETERS: S - decision forest builder, see -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetseed(decisionforestbuilder s, int seedval) { dforest.dfbuildersetseed(s.innerobj, seedval, null); } public static void dfbuildersetseed(decisionforestbuilder s, int seedval, alglib.xparams _params) { dforest.dfbuildersetseed(s.innerobj, seedval, _params); } /************************************************************************* This function sets random decision forest construction algorithm. As for now, only one decision forest construction algorithm is supported - a dense "baseline" RDF algorithm. INPUT PARAMETERS: S - decision forest builder object AlgoType - algorithm type: * 0 = baseline dense RDF OUTPUT PARAMETERS: S - decision forest builder, see -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrdfalgo(decisionforestbuilder s, int algotype) { dforest.dfbuildersetrdfalgo(s.innerobj, algotype, null); } public static void dfbuildersetrdfalgo(decisionforestbuilder s, int algotype, alglib.xparams _params) { dforest.dfbuildersetrdfalgo(s.innerobj, algotype, _params); } /************************************************************************* This function sets split selection algorithm used by decision forest classifier. You may choose several algorithms, with different speed and quality of the results. INPUT PARAMETERS: S - decision forest builder object SplitStrength- split type: * 0 = split at the random position, fastest one * 1 = split at the middle of the range * 2 = strong split at the best point of the range (default) OUTPUT PARAMETERS: S - decision forest builder, see -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrdfsplitstrength(decisionforestbuilder s, int splitstrength) { dforest.dfbuildersetrdfsplitstrength(s.innerobj, splitstrength, null); } public static void dfbuildersetrdfsplitstrength(decisionforestbuilder s, int splitstrength, alglib.xparams _params) { dforest.dfbuildersetrdfsplitstrength(s.innerobj, splitstrength, _params); } /************************************************************************* This function tells decision forest construction algorithm to use Gini impurity based variable importance estimation (also known as MDI). This version of importance estimation algorithm analyzes mean decrease in impurity (MDI) on training sample during splits. The result is divided by impurity at the root node in order to produce estimate in [0,1] range. Such estimates are fast to calculate and beautifully normalized (sum to one) but have following downsides: * They ALWAYS sum to 1.0, even if output is completely unpredictable. I.e. MDI allows to order variables by importance, but does not tell us about "absolute" importances of variables * there exist some bias towards continuous and high-cardinality categorical variables NOTE: informally speaking, MDA (permutation importance) rating answers the question "what part of the model predictive power is ruined by permuting k-th variable?" while MDI tells us "what part of the model predictive power was achieved due to usage of k-th variable". Thus, MDA rates each variable independently at "0 to 1" scale while MDI (and OOB-MDI too) tends to divide "unit amount of importance" between several important variables. If all variables are equally important, they will have same MDI/OOB-MDI rating, equal (for OOB-MDI: roughly equal) to 1/NVars. However, roughly same picture will be produced for the "all variables provide information no one is critical" situation and for the "all variables are critical, drop any one, everything is ruined" situation. Contrary to that, MDA will rate critical variable as ~1.0 important, and important but non-critical variable will have less than unit rating. NOTE: quite an often MDA and MDI return same results. It generally happens on problems with low test set error (a few percents at most) and large enough training set to avoid overfitting. The difference between MDA, MDI and OOB-MDI becomes important only on "hard" tasks with high test set error and/or small training set. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder object. Next call to the forest construction function will produce: * importance estimates in rep.varimportances field * variable ranks in rep.topvars field -- ALGLIB -- Copyright 29.07.2019 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetimportancetrngini(decisionforestbuilder s) { dforest.dfbuildersetimportancetrngini(s.innerobj, null); } public static void dfbuildersetimportancetrngini(decisionforestbuilder s, alglib.xparams _params) { dforest.dfbuildersetimportancetrngini(s.innerobj, _params); } /************************************************************************* This function tells decision forest construction algorithm to use out-of-bag version of Gini variable importance estimation (also known as OOB-MDI). This version of importance estimation algorithm analyzes mean decrease in impurity (MDI) on out-of-bag sample during splits. The result is divided by impurity at the root node in order to produce estimate in [0,1] range. Such estimates are fast to calculate and resistant to overfitting issues (thanks to the out-of-bag estimates used). However, OOB Gini rating has following downsides: * there exist some bias towards continuous and high-cardinality categorical variables * Gini rating allows us to order variables by importance, but it is hard to define importance of the variable by itself. NOTE: informally speaking, MDA (permutation importance) rating answers the question "what part of the model predictive power is ruined by permuting k-th variable?" while MDI tells us "what part of the model predictive power was achieved due to usage of k-th variable". Thus, MDA rates each variable independently at "0 to 1" scale while MDI (and OOB-MDI too) tends to divide "unit amount of importance" between several important variables. If all variables are equally important, they will have same MDI/OOB-MDI rating, equal (for OOB-MDI: roughly equal) to 1/NVars. However, roughly same picture will be produced for the "all variables provide information no one is critical" situation and for the "all variables are critical, drop any one, everything is ruined" situation. Contrary to that, MDA will rate critical variable as ~1.0 important, and important but non-critical variable will have less than unit rating. NOTE: quite an often MDA and MDI return same results. It generally happens on problems with low test set error (a few percents at most) and large enough training set to avoid overfitting. The difference between MDA, MDI and OOB-MDI becomes important only on "hard" tasks with high test set error and/or small training set. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder object. Next call to the forest construction function will produce: * importance estimates in rep.varimportances field * variable ranks in rep.topvars field -- ALGLIB -- Copyright 29.07.2019 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetimportanceoobgini(decisionforestbuilder s) { dforest.dfbuildersetimportanceoobgini(s.innerobj, null); } public static void dfbuildersetimportanceoobgini(decisionforestbuilder s, alglib.xparams _params) { dforest.dfbuildersetimportanceoobgini(s.innerobj, _params); } /************************************************************************* This function tells decision forest construction algorithm to use permutation variable importance estimator (also known as MDA). This version of importance estimation algorithm analyzes mean increase in out-of-bag sum of squared residuals after random permutation of J-th variable. The result is divided by error computed with all variables being perturbed in order to produce R-squared-like estimate in [0,1] range. Such estimate is slower to calculate than Gini-based rating because it needs multiple inference runs for each of variables being studied. ALGLIB uses parallelized and highly optimized algorithm which analyzes path through the decision tree and allows to handle most perturbations in O(1) time; nevertheless, requesting MDA importances may increase forest construction time from 10% to 200% (or more, if you have thousands of variables). However, MDA rating has following benefits over Gini-based ones: * no bias towards specific variable types * ability to directly evaluate "absolute" importance of some variable at "0 to 1" scale (contrary to Gini-based rating, which returns comparative importances). NOTE: informally speaking, MDA (permutation importance) rating answers the question "what part of the model predictive power is ruined by permuting k-th variable?" while MDI tells us "what part of the model predictive power was achieved due to usage of k-th variable". Thus, MDA rates each variable independently at "0 to 1" scale while MDI (and OOB-MDI too) tends to divide "unit amount of importance" between several important variables. If all variables are equally important, they will have same MDI/OOB-MDI rating, equal (for OOB-MDI: roughly equal) to 1/NVars. However, roughly same picture will be produced for the "all variables provide information no one is critical" situation and for the "all variables are critical, drop any one, everything is ruined" situation. Contrary to that, MDA will rate critical variable as ~1.0 important, and important but non-critical variable will have less than unit rating. NOTE: quite an often MDA and MDI return same results. It generally happens on problems with low test set error (a few percents at most) and large enough training set to avoid overfitting. The difference between MDA, MDI and OOB-MDI becomes important only on "hard" tasks with high test set error and/or small training set. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder object. Next call to the forest construction function will produce: * importance estimates in rep.varimportances field * variable ranks in rep.topvars field -- ALGLIB -- Copyright 29.07.2019 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetimportancepermutation(decisionforestbuilder s) { dforest.dfbuildersetimportancepermutation(s.innerobj, null); } public static void dfbuildersetimportancepermutation(decisionforestbuilder s, alglib.xparams _params) { dforest.dfbuildersetimportancepermutation(s.innerobj, _params); } /************************************************************************* This function tells decision forest construction algorithm to skip variable importance estimation. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder object. Next call to the forest construction function will result in forest being built without variable importance estimation. -- ALGLIB -- Copyright 29.07.2019 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetimportancenone(decisionforestbuilder s) { dforest.dfbuildersetimportancenone(s.innerobj, null); } public static void dfbuildersetimportancenone(decisionforestbuilder s, alglib.xparams _params) { dforest.dfbuildersetimportancenone(s.innerobj, _params); } /************************************************************************* This function is an alias for dfbuilderpeekprogress(), left in ALGLIB for backward compatibility reasons. -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static double dfbuildergetprogress(decisionforestbuilder s) { return dforest.dfbuildergetprogress(s.innerobj, null); } public static double dfbuildergetprogress(decisionforestbuilder s, alglib.xparams _params) { return dforest.dfbuildergetprogress(s.innerobj, _params); } /************************************************************************* This function is used to peek into decision forest construction process from some other thread and get current progress indicator. It returns value in [0,1]. INPUT PARAMETERS: S - decision forest builder object used to build forest in some other thread RESULT: progress value, in [0,1] -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static double dfbuilderpeekprogress(decisionforestbuilder s) { return dforest.dfbuilderpeekprogress(s.innerobj, null); } public static double dfbuilderpeekprogress(decisionforestbuilder s, alglib.xparams _params) { return dforest.dfbuilderpeekprogress(s.innerobj, _params); } /************************************************************************* This subroutine builds decision forest according to current settings using dataset internally stored in the builder object. Dense algorithm is used. NOTE: this function uses dense algorithm for forest construction independently from the dataset format (dense or sparse). NOTE: forest built with this function is stored in-memory using 64-bit data structures for offsets/indexes/split values. It is possible to convert forest into more memory-efficient compressed binary representation. Depending on the problem properties, 3.7x-5.7x compression factors are possible. The downsides of compression are (a) slight reduction in the model accuracy and (b) ~1.5x reduction in the inference speed (due to increased complexity of the storage format). See comments on dfbinarycompression() for more info. Default settings are used by the algorithm; you can tweak them with the help of the following functions: * dfbuildersetrfactor() - to control a fraction of the dataset used for subsampling * dfbuildersetrandomvars() - to control number of variables randomly chosen for decision rule creation ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - decision forest builder object NTrees - NTrees>=1, number of trees to train OUTPUT PARAMETERS: DF - decision forest. You can compress this forest to more compact 16-bit representation with dfbinarycompression() Rep - report, see below for information on its fields. === report information produced by forest construction function ========== Decision forest training report includes following information: * training set errors * out-of-bag estimates of errors * variable importance ratings Following fields are used to store information: * training set errors are stored in rep.relclserror, rep.avgce, rep.rmserror, rep.avgerror and rep.avgrelerror * out-of-bag estimates of errors are stored in rep.oobrelclserror, rep.oobavgce, rep.oobrmserror, rep.oobavgerror and rep.oobavgrelerror Variable importance reports, if requested by dfbuildersetimportancegini(), dfbuildersetimportancetrngini() or dfbuildersetimportancepermutation() call, are stored in: * rep.varimportances field stores importance ratings * rep.topvars stores variable indexes ordered from the most important to less important ones You can find more information about report fields in: * comments on dfreport structure * comments on dfbuildersetimportancegini function * comments on dfbuildersetimportancetrngini function * comments on dfbuildersetimportancepermutation function -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuilderbuildrandomforest(decisionforestbuilder s, int ntrees, out decisionforest df, out dfreport rep) { df = new decisionforest(); rep = new dfreport(); dforest.dfbuilderbuildrandomforest(s.innerobj, ntrees, df.innerobj, rep.innerobj, null); } public static void dfbuilderbuildrandomforest(decisionforestbuilder s, int ntrees, out decisionforest df, out dfreport rep, alglib.xparams _params) { df = new decisionforest(); rep = new dfreport(); dforest.dfbuilderbuildrandomforest(s.innerobj, ntrees, df.innerobj, rep.innerobj, _params); } /************************************************************************* This function performs binary compression of the decision forest. Original decision forest produced by the forest builder is stored using 64-bit representation for all numbers - offsets, variable indexes, split points. It is possible to significantly reduce model size by means of: * using compressed dynamic encoding for integers (offsets and variable indexes), which uses just 1 byte to store small ints (less than 128), just 2 bytes for larger values (less than 128^2) and so on * storing floating point numbers using 8-bit exponent and 16-bit mantissa As result, model needs significantly less memory (compression factor depends on variable and class counts). In particular: * NVars<128 and NClasses<128 result in 4.4x-5.7x model size reduction * NVars<16384 and NClasses<128 result in 3.7x-4.5x model size reduction Such storage format performs lossless compression of all integers, but compression of floating point values (split values) is lossy, with roughly 0.01% relative error introduced during rounding. Thus, we recommend you to re-evaluate model accuracy after compression. Another downside of compression is ~1.5x reduction in the inference speed due to necessity of dynamic decompression of the compressed model. INPUT PARAMETERS: DF - decision forest built by forest builder OUTPUT PARAMETERS: DF - replaced by compressed forest RESULT: compression factor (in-RAM size of the compressed model vs than of the uncompressed one), positive number larger than 1.0 -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ public static double dfbinarycompression(decisionforest df) { return dforest.dfbinarycompression(df.innerobj, null); } public static double dfbinarycompression(decisionforest df, alglib.xparams _params) { return dforest.dfbinarycompression(df.innerobj, _params); } /************************************************************************* Inference using decision forest IMPORTANT: this function is thread-unsafe and may modify internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use dftsprocess() with independent thread-local buffers if you need thread-safe evaluation. INPUT PARAMETERS: DF - decision forest model X - input vector, array[NVars] Y - possibly preallocated buffer, reallocated if too small OUTPUT PARAMETERS: Y - result. Regression estimate when solving regression task, vector of posterior probabilities for classification task. See also DFProcessI. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static void dfprocess(decisionforest df, double[] x, ref double[] y) { dforest.dfprocess(df.innerobj, x, ref y, null); } public static void dfprocess(decisionforest df, double[] x, ref double[] y, alglib.xparams _params) { dforest.dfprocess(df.innerobj, x, ref y, _params); } /************************************************************************* 'interactive' variant of DFProcess for languages like Python which support constructs like "Y = DFProcessI(DF,X)" and interactive mode of interpreter This function allocates new array on each call, so it is significantly slower than its 'non-interactive' counterpart, but it is more convenient when you call it from command line. IMPORTANT: this function is thread-unsafe and may modify internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use dftsprocess() with independent thread-local buffers if you need thread-safe evaluation. -- ALGLIB -- Copyright 28.02.2010 by Bochkanov Sergey *************************************************************************/ public static void dfprocessi(decisionforest df, double[] x, out double[] y) { y = new double[0]; dforest.dfprocessi(df.innerobj, x, ref y, null); } public static void dfprocessi(decisionforest df, double[] x, out double[] y, alglib.xparams _params) { y = new double[0]; dforest.dfprocessi(df.innerobj, x, ref y, _params); } /************************************************************************* This function returns first component of the inferred vector (i.e. one with index #0). It is a convenience wrapper for dfprocess() intended for either: * 1-dimensional regression problems * 2-class classification problems In the former case this function returns inference result as scalar, which is definitely more convenient that wrapping it as vector. In the latter case it returns probability of object belonging to class #0. If you call it for anything different from two cases above, it will work as defined, i.e. return y[0], although it is of less use in such cases. IMPORTANT: this function is thread-unsafe and modifies internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use dftsprocess() with independent thread-local buffers, if you need thread-safe evaluation. INPUT PARAMETERS: Model - DF model X - input vector, array[0..NVars-1]. RESULT: Y[0] -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static double dfprocess0(decisionforest model, double[] x) { return dforest.dfprocess0(model.innerobj, x, null); } public static double dfprocess0(decisionforest model, double[] x, alglib.xparams _params) { return dforest.dfprocess0(model.innerobj, x, _params); } /************************************************************************* This function returns most probable class number for an input X. It is same as calling dfprocess(model,x,y), then determining i=argmax(y[i]) and returning i. A class number in [0,NOut) range in returned for classification problems, -1 is returned when this function is called for regression problems. IMPORTANT: this function is thread-unsafe and modifies internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use dftsprocess() with independent thread-local buffers, if you need thread-safe evaluation. INPUT PARAMETERS: Model - decision forest model X - input vector, array[0..NVars-1]. RESULT: class number, -1 for regression tasks -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static int dfclassify(decisionforest model, double[] x) { return dforest.dfclassify(model.innerobj, x, null); } public static int dfclassify(decisionforest model, double[] x, alglib.xparams _params) { return dforest.dfclassify(model.innerobj, x, _params); } /************************************************************************* Inference using decision forest Thread-safe procesing using external buffer for temporaries. This function is thread-safe (i.e . you can use same DF model from multiple threads) as long as you use different buffer objects for different threads. INPUT PARAMETERS: DF - decision forest model Buf - buffer object, must be allocated specifically for this model with dfcreatebuffer(). X - input vector, array[NVars] Y - possibly preallocated buffer, reallocated if too small OUTPUT PARAMETERS: Y - result. Regression estimate when solving regression task, vector of posterior probabilities for classification task. See also DFProcessI. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static void dftsprocess(decisionforest df, decisionforestbuffer buf, double[] x, ref double[] y) { dforest.dftsprocess(df.innerobj, buf.innerobj, x, ref y, null); } public static void dftsprocess(decisionforest df, decisionforestbuffer buf, double[] x, ref double[] y, alglib.xparams _params) { dforest.dftsprocess(df.innerobj, buf.innerobj, x, ref y, _params); } /************************************************************************* Relative classification error on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: percent of incorrectly classified cases. Zero if model solves regression task. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfrelclserror(decisionforest df, double[,] xy, int npoints) { return dforest.dfrelclserror(df.innerobj, xy, npoints, null); } public static double dfrelclserror(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { return dforest.dfrelclserror(df.innerobj, xy, npoints, _params); } /************************************************************************* Average cross-entropy (in bits per element) on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: CrossEntropy/(NPoints*LN(2)). Zero if model solves regression task. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfavgce(decisionforest df, double[,] xy, int npoints) { return dforest.dfavgce(df.innerobj, xy, npoints, null); } public static double dfavgce(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { return dforest.dfavgce(df.innerobj, xy, npoints, _params); } /************************************************************************* RMS error on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: root mean square error. Its meaning for regression task is obvious. As for classification task, RMS error means error when estimating posterior probabilities. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfrmserror(decisionforest df, double[,] xy, int npoints) { return dforest.dfrmserror(df.innerobj, xy, npoints, null); } public static double dfrmserror(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { return dforest.dfrmserror(df.innerobj, xy, npoints, _params); } /************************************************************************* Average error on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: Its meaning for regression task is obvious. As for classification task, it means average error when estimating posterior probabilities. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfavgerror(decisionforest df, double[,] xy, int npoints) { return dforest.dfavgerror(df.innerobj, xy, npoints, null); } public static double dfavgerror(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { return dforest.dfavgerror(df.innerobj, xy, npoints, _params); } /************************************************************************* Average relative error on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: Its meaning for regression task is obvious. As for classification task, it means average relative error when estimating posterior probability of belonging to the correct class. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfavgrelerror(decisionforest df, double[,] xy, int npoints) { return dforest.dfavgrelerror(df.innerobj, xy, npoints, null); } public static double dfavgrelerror(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { return dforest.dfavgrelerror(df.innerobj, xy, npoints, _params); } /************************************************************************* This subroutine builds random decision forest. --------- DEPRECATED VERSION! USE DECISION FOREST BUILDER OBJECT --------- -- ALGLIB -- Copyright 19.02.2009 by Bochkanov Sergey *************************************************************************/ public static void dfbuildrandomdecisionforest(double[,] xy, int npoints, int nvars, int nclasses, int ntrees, double r, out int info, out decisionforest df, out dfreport rep) { info = 0; df = new decisionforest(); rep = new dfreport(); dforest.dfbuildrandomdecisionforest(xy, npoints, nvars, nclasses, ntrees, r, ref info, df.innerobj, rep.innerobj, null); } public static void dfbuildrandomdecisionforest(double[,] xy, int npoints, int nvars, int nclasses, int ntrees, double r, out int info, out decisionforest df, out dfreport rep, alglib.xparams _params) { info = 0; df = new decisionforest(); rep = new dfreport(); dforest.dfbuildrandomdecisionforest(xy, npoints, nvars, nclasses, ntrees, r, ref info, df.innerobj, rep.innerobj, _params); } /************************************************************************* This subroutine builds random decision forest. --------- DEPRECATED VERSION! USE DECISION FOREST BUILDER OBJECT --------- -- ALGLIB -- Copyright 19.02.2009 by Bochkanov Sergey *************************************************************************/ public static void dfbuildrandomdecisionforestx1(double[,] xy, int npoints, int nvars, int nclasses, int ntrees, int nrndvars, double r, out int info, out decisionforest df, out dfreport rep) { info = 0; df = new decisionforest(); rep = new dfreport(); dforest.dfbuildrandomdecisionforestx1(xy, npoints, nvars, nclasses, ntrees, nrndvars, r, ref info, df.innerobj, rep.innerobj, null); } public static void dfbuildrandomdecisionforestx1(double[,] xy, int npoints, int nvars, int nclasses, int ntrees, int nrndvars, double r, out int info, out decisionforest df, out dfreport rep, alglib.xparams _params) { info = 0; df = new decisionforest(); rep = new dfreport(); dforest.dfbuildrandomdecisionforestx1(xy, npoints, nvars, nclasses, ntrees, nrndvars, r, ref info, df.innerobj, rep.innerobj, _params); } } public partial class alglib { /************************************************************************* Buffer object which is used to perform various requests (usually model inference) in the multithreaded mode (multiple threads working with same KNN object). This object should be created with KNNCreateBuffer(). *************************************************************************/ public class knnbuffer : alglibobject { // // Public declarations // public knnbuffer() { _innerobj = new knn.knnbuffer(); } public override alglib.alglibobject make_copy() { return new knnbuffer((knn.knnbuffer)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private knn.knnbuffer _innerobj; public knn.knnbuffer innerobj { get { return _innerobj; } } public knnbuffer(knn.knnbuffer obj) { _innerobj = obj; } } /************************************************************************* A KNN builder object; this object encapsulates dataset and all related settings, it is used to create an actual instance of KNN model. *************************************************************************/ public class knnbuilder : alglibobject { // // Public declarations // public knnbuilder() { _innerobj = new knn.knnbuilder(); } public override alglib.alglibobject make_copy() { return new knnbuilder((knn.knnbuilder)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private knn.knnbuilder _innerobj; public knn.knnbuilder innerobj { get { return _innerobj; } } public knnbuilder(knn.knnbuilder obj) { _innerobj = obj; } } /************************************************************************* KNN model, can be used for classification or regression *************************************************************************/ public class knnmodel : alglibobject { // // Public declarations // public knnmodel() { _innerobj = new knn.knnmodel(); } public override alglib.alglibobject make_copy() { return new knnmodel((knn.knnmodel)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private knn.knnmodel _innerobj; public knn.knnmodel innerobj { get { return _innerobj; } } public knnmodel(knn.knnmodel obj) { _innerobj = obj; } } /************************************************************************* KNN training report. Following fields store training set errors: * relclserror - fraction of misclassified cases, [0,1] * avgce - average cross-entropy in bits per symbol * rmserror - root-mean-square error * avgerror - average error * avgrelerror - average relative error For classification problems: * RMS, AVG and AVGREL errors are calculated for posterior probabilities For regression problems: * RELCLS and AVGCE errors are zero *************************************************************************/ public class knnreport : alglibobject { // // Public declarations // public double relclserror { get { return _innerobj.relclserror; } set { _innerobj.relclserror = value; } } public double avgce { get { return _innerobj.avgce; } set { _innerobj.avgce = value; } } public double rmserror { get { return _innerobj.rmserror; } set { _innerobj.rmserror = value; } } public double avgerror { get { return _innerobj.avgerror; } set { _innerobj.avgerror = value; } } public double avgrelerror { get { return _innerobj.avgrelerror; } set { _innerobj.avgrelerror = value; } } public knnreport() { _innerobj = new knn.knnreport(); } public override alglib.alglibobject make_copy() { return new knnreport((knn.knnreport)_innerobj.make_copy()); } // // Although some of declarations below are public, you should not use them // They are intended for internal use only // private knn.knnreport _innerobj; public knn.knnreport innerobj { get { return _innerobj; } } public knnreport(knn.knnreport obj) { _innerobj = obj; } } /************************************************************************* This function serializes data structure to string. Important properties of s_out: * it contains alphanumeric characters, dots, underscores, minus signs * these symbols are grouped into words, which are separated by spaces and Windows-style (CR+LF) newlines * although serializer uses spaces and CR+LF as separators, you can replace any separator character by arbitrary combination of spaces, tabs, Windows or Unix newlines. It allows flexible reformatting of the string in case you want to include it into text or XML file. But you should not insert separators into the middle of the "words" nor you should change case of letters. * s_out can be freely moved between 32-bit and 64-bit systems, little and big endian machines, and so on. You can serialize structure on 32-bit machine and unserialize it on 64-bit one (or vice versa), or serialize it on SPARC and unserialize on x86. You can also serialize it in C# version of ALGLIB and unserialize in C++ one, and vice versa. *************************************************************************/ public static void knnserialize(knnmodel obj, out string s_out) { alglib.serializer s = new alglib.serializer(); s.alloc_start(); knn.knnalloc(s, obj.innerobj, null); s.sstart_str(); knn.knnserialize(s, obj.innerobj, null); s.stop(); s_out = s.get_string(); } /************************************************************************* This function unserializes data structure from string. *************************************************************************/ public static void knnunserialize(string s_in, out knnmodel obj) { alglib.serializer s = new alglib.serializer(); obj = new knnmodel(); s.ustart_str(s_in); knn.knnunserialize(s, obj.innerobj, null); s.stop(); } /************************************************************************* This function serializes data structure to stream. Data stream generated by this function is same as string representation generated by string version of serializer - alphanumeric characters, dots, underscores, minus signs, which are grouped into words separated by spaces and CR+LF. We recommend you to read comments on string version of serializer to find out more about serialization of AlGLIB objects. *************************************************************************/ public static void knnserialize(knnmodel obj, System.IO.Stream stream_out) { alglib.serializer s = new alglib.serializer(); s.alloc_start(); knn.knnalloc(s, obj.innerobj, null); s.sstart_stream(stream_out); knn.knnserialize(s, obj.innerobj, null); s.stop(); } /************************************************************************* This function unserializes data structure from stream. *************************************************************************/ public static void knnunserialize(System.IO.Stream stream_in, out knnmodel obj) { alglib.serializer s = new alglib.serializer(); obj = new knnmodel(); s.ustart_stream(stream_in); knn.knnunserialize(s, obj.innerobj, null); s.stop(); } /************************************************************************* This function creates buffer structure which can be used to perform parallel KNN requests. KNN subpackage provides two sets of computing functions - ones which use internal buffer of KNN model (these functions are single-threaded because they use same buffer, which can not shared between threads), and ones which use external buffer. This function is used to initialize external buffer. INPUT PARAMETERS Model - KNN model which is associated with newly created buffer OUTPUT PARAMETERS Buf - external buffer. IMPORTANT: buffer object should be used only with model which was used to initialize buffer. Any attempt to use buffer with different object is dangerous - you may get integrity check failure (exception) because sizes of internal arrays do not fit to dimensions of the model structure. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knncreatebuffer(knnmodel model, out knnbuffer buf) { buf = new knnbuffer(); knn.knncreatebuffer(model.innerobj, buf.innerobj, null); } public static void knncreatebuffer(knnmodel model, out knnbuffer buf, alglib.xparams _params) { buf = new knnbuffer(); knn.knncreatebuffer(model.innerobj, buf.innerobj, _params); } /************************************************************************* This subroutine creates KNNBuilder object which is used to train KNN models. By default, new builder stores empty dataset and some reasonable default settings. At the very least, you should specify dataset prior to building KNN model. You can also tweak settings of the model construction algorithm (recommended, although default settings should work well). Following actions are mandatory: * calling knnbuildersetdataset() to specify dataset * calling knnbuilderbuildknnmodel() to build KNN model using current dataset and default settings Additionally, you may call: * knnbuildersetnorm() to change norm being used INPUT PARAMETERS: none OUTPUT PARAMETERS: S - KNN builder -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnbuildercreate(out knnbuilder s) { s = new knnbuilder(); knn.knnbuildercreate(s.innerobj, null); } public static void knnbuildercreate(out knnbuilder s, alglib.xparams _params) { s = new knnbuilder(); knn.knnbuildercreate(s.innerobj, _params); } /************************************************************************* Specifies regression problem (one or more continuous output variables are predicted). There also exists "classification" version of this function. This subroutine adds dense dataset to the internal storage of the builder object. Specifying your dataset in the dense format means that the dense version of the KNN construction algorithm will be invoked. INPUT PARAMETERS: S - KNN builder object XY - array[NPoints,NVars+NOut] (note: actual size can be larger, only leading part is used anyway), dataset: * first NVars elements of each row store values of the independent variables * next NOut elements store values of the dependent variables NPoints - number of rows in the dataset, NPoints>=1 NVars - number of independent variables, NVars>=1 NOut - number of dependent variables, NOut>=1 OUTPUT PARAMETERS: S - KNN builder -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnbuildersetdatasetreg(knnbuilder s, double[,] xy, int npoints, int nvars, int nout) { knn.knnbuildersetdatasetreg(s.innerobj, xy, npoints, nvars, nout, null); } public static void knnbuildersetdatasetreg(knnbuilder s, double[,] xy, int npoints, int nvars, int nout, alglib.xparams _params) { knn.knnbuildersetdatasetreg(s.innerobj, xy, npoints, nvars, nout, _params); } /************************************************************************* Specifies classification problem (two or more classes are predicted). There also exists "regression" version of this function. This subroutine adds dense dataset to the internal storage of the builder object. Specifying your dataset in the dense format means that the dense version of the KNN construction algorithm will be invoked. INPUT PARAMETERS: S - KNN builder object XY - array[NPoints,NVars+1] (note: actual size can be larger, only leading part is used anyway), dataset: * first NVars elements of each row store values of the independent variables * next element stores class index, in [0,NClasses) NPoints - number of rows in the dataset, NPoints>=1 NVars - number of independent variables, NVars>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: S - KNN builder -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnbuildersetdatasetcls(knnbuilder s, double[,] xy, int npoints, int nvars, int nclasses) { knn.knnbuildersetdatasetcls(s.innerobj, xy, npoints, nvars, nclasses, null); } public static void knnbuildersetdatasetcls(knnbuilder s, double[,] xy, int npoints, int nvars, int nclasses, alglib.xparams _params) { knn.knnbuildersetdatasetcls(s.innerobj, xy, npoints, nvars, nclasses, _params); } /************************************************************************* This function sets norm type used for neighbor search. INPUT PARAMETERS: S - decision forest builder object NormType - norm type: * 0 inf-norm * 1 1-norm * 2 Euclidean norm (default) OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnbuildersetnorm(knnbuilder s, int nrmtype) { knn.knnbuildersetnorm(s.innerobj, nrmtype, null); } public static void knnbuildersetnorm(knnbuilder s, int nrmtype, alglib.xparams _params) { knn.knnbuildersetnorm(s.innerobj, nrmtype, _params); } /************************************************************************* This subroutine builds KNN model according to current settings, using dataset internally stored in the builder object. The model being built performs inference using Eps-approximate K nearest neighbors search algorithm, with: * K=1, Eps=0 corresponding to the "nearest neighbor algorithm" * K>1, Eps=0 corresponding to the "K nearest neighbors algorithm" * K>=1, Eps>0 corresponding to "approximate nearest neighbors algorithm" An approximate KNN is a good option for high-dimensional datasets (exact KNN works slowly when dimensions count grows). An ALGLIB implementation of kd-trees is used to perform k-nn searches. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - KNN builder object K - number of neighbors to search for, K>=1 Eps - approximation factor: * Eps=0 means that exact kNN search is performed * Eps>0 means that (1+Eps)-approximate search is performed OUTPUT PARAMETERS: Model - KNN model Rep - report -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnbuilderbuildknnmodel(knnbuilder s, int k, double eps, out knnmodel model, out knnreport rep) { model = new knnmodel(); rep = new knnreport(); knn.knnbuilderbuildknnmodel(s.innerobj, k, eps, model.innerobj, rep.innerobj, null); } public static void knnbuilderbuildknnmodel(knnbuilder s, int k, double eps, out knnmodel model, out knnreport rep, alglib.xparams _params) { model = new knnmodel(); rep = new knnreport(); knn.knnbuilderbuildknnmodel(s.innerobj, k, eps, model.innerobj, rep.innerobj, _params); } /************************************************************************* Changing search settings of KNN model. K and EPS parameters of KNN (AKNN) search are specified during model construction. However, plain KNN algorithm with Euclidean distance allows you to change them at any moment. NOTE: future versions of KNN model may support advanced versions of KNN, such as NCA or LMNN. It is possible that such algorithms won't allow you to change search settings on the fly. If you call this function for an algorithm which does not support on-the-fly changes, it will throw an exception. INPUT PARAMETERS: Model - KNN model K - K>=1, neighbors count EPS - accuracy of the EPS-approximate NN search. Set to 0.0, if you want to perform "classic" KNN search. Specify larger values if you need to speed-up high-dimensional KNN queries. OUTPUT PARAMETERS: nothing on success, exception on failure -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnrewritekeps(knnmodel model, int k, double eps) { knn.knnrewritekeps(model.innerobj, k, eps, null); } public static void knnrewritekeps(knnmodel model, int k, double eps, alglib.xparams _params) { knn.knnrewritekeps(model.innerobj, k, eps, _params); } /************************************************************************* Inference using KNN model. See also knnprocess0(), knnprocessi() and knnclassify() for options with a bit more convenient interface. IMPORTANT: this function is thread-unsafe and modifies internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use knntsprocess() with independent thread-local buffers, if you need thread-safe evaluation. INPUT PARAMETERS: Model - KNN model X - input vector, array[0..NVars-1]. Y - possible preallocated buffer. Reused if long enough. OUTPUT PARAMETERS: Y - result. Regression estimate when solving regression task, vector of posterior probabilities for classification task. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnprocess(knnmodel model, double[] x, ref double[] y) { knn.knnprocess(model.innerobj, x, ref y, null); } public static void knnprocess(knnmodel model, double[] x, ref double[] y, alglib.xparams _params) { knn.knnprocess(model.innerobj, x, ref y, _params); } /************************************************************************* This function returns first component of the inferred vector (i.e. one with index #0). It is a convenience wrapper for knnprocess() intended for either: * 1-dimensional regression problems * 2-class classification problems In the former case this function returns inference result as scalar, which is definitely more convenient that wrapping it as vector. In the latter case it returns probability of object belonging to class #0. If you call it for anything different from two cases above, it will work as defined, i.e. return y[0], although it is of less use in such cases. IMPORTANT: this function is thread-unsafe and modifies internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use knntsprocess() with independent thread-local buffers, if you need thread-safe evaluation. INPUT PARAMETERS: Model - KNN model X - input vector, array[0..NVars-1]. RESULT: Y[0] -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static double knnprocess0(knnmodel model, double[] x) { return knn.knnprocess0(model.innerobj, x, null); } public static double knnprocess0(knnmodel model, double[] x, alglib.xparams _params) { return knn.knnprocess0(model.innerobj, x, _params); } /************************************************************************* This function returns most probable class number for an input X. It is same as calling knnprocess(model,x,y), then determining i=argmax(y[i]) and returning i. A class number in [0,NOut) range in returned for classification problems, -1 is returned when this function is called for regression problems. IMPORTANT: this function is thread-unsafe and modifies internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use knntsprocess() with independent thread-local buffers, if you need thread-safe evaluation. INPUT PARAMETERS: Model - KNN model X - input vector, array[0..NVars-1]. RESULT: class number, -1 for regression tasks -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static int knnclassify(knnmodel model, double[] x) { return knn.knnclassify(model.innerobj, x, null); } public static int knnclassify(knnmodel model, double[] x, alglib.xparams _params) { return knn.knnclassify(model.innerobj, x, _params); } /************************************************************************* 'interactive' variant of knnprocess() for languages like Python which support constructs like "y = knnprocessi(model,x)" and interactive mode of the interpreter. This function allocates new array on each call, so it is significantly slower than its 'non-interactive' counterpart, but it is more convenient when you call it from command line. IMPORTANT: this function is thread-unsafe and may modify internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use knntsprocess() with independent thread-local buffers if you need thread-safe evaluation. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnprocessi(knnmodel model, double[] x, out double[] y) { y = new double[0]; knn.knnprocessi(model.innerobj, x, ref y, null); } public static void knnprocessi(knnmodel model, double[] x, out double[] y, alglib.xparams _params) { y = new double[0]; knn.knnprocessi(model.innerobj, x, ref y, _params); } /************************************************************************* Thread-safe procesing using external buffer for temporaries. This function is thread-safe (i.e . you can use same KNN model from multiple threads) as long as you use different buffer objects for different threads. INPUT PARAMETERS: Model - KNN model Buf - buffer object, must be allocated specifically for this model with knncreatebuffer(). X - input vector, array[NVars] OUTPUT PARAMETERS: Y - result, array[NOut]. Regression estimate when solving regression task, vector of posterior probabilities for a classification task. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knntsprocess(knnmodel model, knnbuffer buf, double[] x, ref double[] y) { knn.knntsprocess(model.innerobj, buf.innerobj, x, ref y, null); } public static void knntsprocess(knnmodel model, knnbuffer buf, double[] x, ref double[] y, alglib.xparams _params) { knn.knntsprocess(model.innerobj, buf.innerobj, x, ref y, _params); } /************************************************************************* Relative classification error on the test set INPUT PARAMETERS: Model - KNN model XY - test set NPoints - test set size RESULT: percent of incorrectly classified cases. Zero if model solves regression task. NOTE: if you need several different kinds of error metrics, it is better to use knnallerrors() which computes all error metric with just one pass over dataset. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static double knnrelclserror(knnmodel model, double[,] xy, int npoints) { return knn.knnrelclserror(model.innerobj, xy, npoints, null); } public static double knnrelclserror(knnmodel model, double[,] xy, int npoints, alglib.xparams _params) { return knn.knnrelclserror(model.innerobj, xy, npoints, _params); } /************************************************************************* Average cross-entropy (in bits per element) on the test set INPUT PARAMETERS: Model - KNN model XY - test set NPoints - test set size RESULT: CrossEntropy/NPoints. Zero if model solves regression task. NOTE: the cross-entropy metric is too unstable when used to evaluate KNN models (such models can report exactly zero probabilities), so we do not recommend using it. NOTE: if you need several different kinds of error metrics, it is better to use knnallerrors() which computes all error metric with just one pass over dataset. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static double knnavgce(knnmodel model, double[,] xy, int npoints) { return knn.knnavgce(model.innerobj, xy, npoints, null); } public static double knnavgce(knnmodel model, double[,] xy, int npoints, alglib.xparams _params) { return knn.knnavgce(model.innerobj, xy, npoints, _params); } /************************************************************************* RMS error on the test set. Its meaning for regression task is obvious. As for classification problems, RMS error means error when estimating posterior probabilities. INPUT PARAMETERS: Model - KNN model XY - test set NPoints - test set size RESULT: root mean square error. NOTE: if you need several different kinds of error metrics, it is better to use knnallerrors() which computes all error metric with just one pass over dataset. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static double knnrmserror(knnmodel model, double[,] xy, int npoints) { return knn.knnrmserror(model.innerobj, xy, npoints, null); } public static double knnrmserror(knnmodel model, double[,] xy, int npoints, alglib.xparams _params) { return knn.knnrmserror(model.innerobj, xy, npoints, _params); } /************************************************************************* Average error on the test set Its meaning for regression task is obvious. As for classification problems, average error means error when estimating posterior probabilities. INPUT PARAMETERS: Model - KNN model XY - test set NPoints - test set size RESULT: average error NOTE: if you need several different kinds of error metrics, it is better to use knnallerrors() which computes all error metric with just one pass over dataset. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static double knnavgerror(knnmodel model, double[,] xy, int npoints) { return knn.knnavgerror(model.innerobj, xy, npoints, null); } public static double knnavgerror(knnmodel model, double[,] xy, int npoints, alglib.xparams _params) { return knn.knnavgerror(model.innerobj, xy, npoints, _params); } /************************************************************************* Average relative error on the test set Its meaning for regression task is obvious. As for classification problems, average relative error means error when estimating posterior probabilities. INPUT PARAMETERS: Model - KNN model XY - test set NPoints - test set size RESULT: average relative error NOTE: if you need several different kinds of error metrics, it is better to use knnallerrors() which computes all error metric with just one pass over dataset. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static double knnavgrelerror(knnmodel model, double[,] xy, int npoints) { return knn.knnavgrelerror(model.innerobj, xy, npoints, null); } public static double knnavgrelerror(knnmodel model, double[,] xy, int npoints, alglib.xparams _params) { return knn.knnavgrelerror(model.innerobj, xy, npoints, _params); } /************************************************************************* Calculates all kinds of errors for the model in one call. INPUT PARAMETERS: Model - KNN model XY - test set: * one row per point * first NVars columns store independent variables * depending on problem type: * next column stores class number in [0,NClasses) - for classification problems * next NOut columns store dependent variables - for regression problems NPoints - test set size, NPoints>=0 OUTPUT PARAMETERS: Rep - following fields are loaded with errors for both regression and classification models: * rep.rmserror - RMS error for the output * rep.avgerror - average error * rep.avgrelerror - average relative error following fields are set only for classification models, zero for regression ones: * relclserror - relative classification error, in [0,1] * avgce - average cross-entropy in bits per dataset entry NOTE: the cross-entropy metric is too unstable when used to evaluate KNN models (such models can report exactly zero probabilities), so we do not recommend using it. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnallerrors(knnmodel model, double[,] xy, int npoints, out knnreport rep) { rep = new knnreport(); knn.knnallerrors(model.innerobj, xy, npoints, rep.innerobj, null); } public static void knnallerrors(knnmodel model, double[,] xy, int npoints, out knnreport rep, alglib.xparams _params) { rep = new knnreport(); knn.knnallerrors(model.innerobj, xy, npoints, rep.innerobj, _params); } } public partial class alglib { /************************************************************************* k-means++ clusterization. Backward compatibility function, we recommend to use CLUSTERING subpackage as better replacement. -- ALGLIB -- Copyright 21.03.2009 by Bochkanov Sergey *************************************************************************/ public static void kmeansgenerate(double[,] xy, int npoints, int nvars, int k, int restarts, out int info, out double[,] c, out int[] xyc) { info = 0; c = new double[0,0]; xyc = new int[0]; datacomp.kmeansgenerate(xy, npoints, nvars, k, restarts, ref info, ref c, ref xyc, null); } public static void kmeansgenerate(double[,] xy, int npoints, int nvars, int k, int restarts, out int info, out double[,] c, out int[] xyc, alglib.xparams _params) { info = 0; c = new double[0,0]; xyc = new int[0]; datacomp.kmeansgenerate(xy, npoints, nvars, k, restarts, ref info, ref c, ref xyc, _params); } } public partial class alglib { public class pca { /************************************************************************* Principal components analysis This function builds orthogonal basis where first axis corresponds to direction with maximum variance, second axis maximizes variance in the subspace orthogonal to first axis and so on. This function builds FULL basis, i.e. returns N vectors corresponding to ALL directions, no matter how informative. If you need just a few (say, 10 or 50) of the most important directions, you may find it faster to use one of the reduced versions: * pcatruncatedsubspace() - for subspace iteration based method It should be noted that, unlike LDA, PCA does not use class labels. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: X - dataset, array[0..NPoints-1,0..NVars-1]. matrix contains ONLY INDEPENDENT VARIABLES. NPoints - dataset size, NPoints>=0 NVars - number of independent variables, NVars>=1 OUTPUT PARAMETERS: Info - return code: * -4, if SVD subroutine haven't converged * -1, if wrong parameters has been passed (NPoints<0, NVars<1) * 1, if task is solved S2 - array[0..NVars-1]. variance values corresponding to basis vectors. V - array[0..NVars-1,0..NVars-1] matrix, whose columns store basis vectors. -- ALGLIB -- Copyright 25.08.2008 by Bochkanov Sergey *************************************************************************/ public static void pcabuildbasis(double[,] x, int npoints, int nvars, ref int info, ref double[] s2, ref double[,] v, alglib.xparams _params) { double[,] a = new double[0,0]; double[,] u = new double[0,0]; double[,] vt = new double[0,0]; double[] m = new double[0]; double[] t = new double[0]; int i = 0; int j = 0; double mean = 0; double variance = 0; double skewness = 0; double kurtosis = 0; int i_ = 0; info = 0; s2 = new double[0]; v = new double[0,0]; // // Check input data // if( npoints<0 || nvars<1 ) { info = -1; return; } info = 1; // // Special case: NPoints=0 // if( npoints==0 ) { s2 = new double[nvars]; v = new double[nvars, nvars]; for(i=0; i<=nvars-1; i++) { s2[i] = 0; } for(i=0; i<=nvars-1; i++) { for(j=0; j<=nvars-1; j++) { if( i==j ) { v[i,j] = 1; } else { v[i,j] = 0; } } } return; } // // Calculate means // m = new double[nvars]; t = new double[npoints]; for(j=0; j<=nvars-1; j++) { for(i_=0; i_<=npoints-1;i_++) { t[i_] = x[i_,j]; } basestat.samplemoments(t, npoints, ref mean, ref variance, ref skewness, ref kurtosis, _params); m[j] = mean; } // // Center, apply SVD, prepare output // a = new double[Math.Max(npoints, nvars), nvars]; for(i=0; i<=npoints-1; i++) { for(i_=0; i_<=nvars-1;i_++) { a[i,i_] = x[i,i_]; } for(i_=0; i_<=nvars-1;i_++) { a[i,i_] = a[i,i_] - m[i_]; } } for(i=npoints; i<=nvars-1; i++) { for(j=0; j<=nvars-1; j++) { a[i,j] = 0; } } if( !svd.rmatrixsvd(a, Math.Max(npoints, nvars), nvars, 0, 1, 2, ref s2, ref u, ref vt, _params) ) { info = -4; return; } if( npoints!=1 ) { for(i=0; i<=nvars-1; i++) { s2[i] = math.sqr(s2[i])/(npoints-1); } } v = new double[nvars, nvars]; blas.copyandtranspose(vt, 0, nvars-1, 0, nvars-1, ref v, 0, nvars-1, 0, nvars-1, _params); } /************************************************************************* Principal components analysis This function performs truncated PCA, i.e. returns just a few most important directions. Internally it uses iterative eigensolver which is very efficient when only a minor fraction of full basis is required. Thus, if you need full basis, it is better to use pcabuildbasis() function. It should be noted that, unlike LDA, PCA does not use class labels. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: X - dataset, array[0..NPoints-1,0..NVars-1]. matrix contains ONLY INDEPENDENT VARIABLES. NPoints - dataset size, NPoints>=0 NVars - number of independent variables, NVars>=1 NNeeded - number of requested components, in [1,NVars] range; this function is efficient only for NNeeded<=0, "PCATruncatedSubspace: npoints<0"); alglib.ap.assert(nvars>=1, "PCATruncatedSubspace: nvars<1"); alglib.ap.assert(nneeded>0, "PCATruncatedSubspace: nneeded<1"); alglib.ap.assert(nneeded<=nvars, "PCATruncatedSubspace: nneeded>nvars"); alglib.ap.assert(maxits>=0, "PCATruncatedSubspace: maxits<0"); alglib.ap.assert(math.isfinite(eps) && (double)(eps)>=(double)(0), "PCATruncatedSubspace: eps<0 or is not finite"); alglib.ap.assert(alglib.ap.rows(x)>=npoints, "PCATruncatedSubspace: rows(x)=nvars || npoints==0, "PCATruncatedSubspace: cols(x)=0 NVars - number of independent variables, NVars>=1 NNeeded - number of requested components, in [1,NVars] range; this function is efficient only for NNeeded<=0, "PCATruncatedSubspaceSparse: npoints<0"); alglib.ap.assert(nvars>=1, "PCATruncatedSubspaceSparse: nvars<1"); alglib.ap.assert(nneeded>0, "PCATruncatedSubspaceSparse: nneeded<1"); alglib.ap.assert(nneeded<=nvars, "PCATruncatedSubspaceSparse: nneeded>nvars"); alglib.ap.assert(maxits>=0, "PCATruncatedSubspaceSparse: maxits<0"); alglib.ap.assert(math.isfinite(eps) && (double)(eps)>=(double)(0), "PCATruncatedSubspaceSparse: eps<0 or is not finite"); if( npoints>0 ) { alglib.ap.assert(sparse.sparsegetnrows(x, _params)==npoints, "PCATruncatedSubspaceSparse: rows(x)!=npoints"); alglib.ap.assert(sparse.sparsegetncols(x, _params)==nvars, "PCATruncatedSubspaceSparse: cols(x)!=nvars"); } // // Special case: NPoints=0 // if( npoints==0 ) { s2 = new double[nneeded]; v = new double[nvars, nneeded]; for(i=0; i<=nvars-1; i++) { s2[i] = 0; } for(i=0; i<=nvars-1; i++) { for(j=0; j<=nneeded-1; j++) { if( i==j ) { v[i,j] = 1; } else { v[i,j] = 0; } } } return; } // // If input data are not in CRS format, perform conversion to CRS // if( !sparse.sparseiscrs(x, _params) ) { sparse.sparsecopytocrs(x, xcrs, _params); pcatruncatedsubspacesparse(xcrs, npoints, nvars, nneeded, eps, maxits, ref s2, ref v, _params); return; } // // Initialize parameters, prepare buffers // b1 = new double[npoints]; z1 = new double[nvars]; if( (double)(eps)==(double)(0) && maxits==0 ) { eps = 1.0E-6; } if( maxits==0 ) { maxits = 50+2*nvars; } // // Calculate mean values // vv = (double)1/(double)npoints; for(i=0; i<=npoints-1; i++) { b1[i] = vv; } sparse.sparsemtv(x, b1, ref means, _params); // // Find eigenvalues with subspace iteration solver // evd.eigsubspacecreate(nvars, nneeded, solver, _params); evd.eigsubspacesetcond(solver, eps, maxits, _params); evd.eigsubspaceoocstart(solver, 0, _params); while( evd.eigsubspaceooccontinue(solver, _params) ) { alglib.ap.assert(solver.requesttype==0, "PCATruncatedSubspace: integrity check failed"); for(k=0; k<=solver.requestsize-1; k++) { // // Calculate B1=(X-meansX)*Zk // for(i_=0; i_<=nvars-1;i_++) { z1[i_] = solver.x[i_,k]; } sparse.sparsemv(x, z1, ref b1, _params); vv = 0.0; for(i_=0; i_<=nvars-1;i_++) { vv += solver.x[i_,k]*means[i_]; } for(i=0; i<=npoints-1; i++) { b1[i] = b1[i]-vv; } // // Calculate (X-meansX)^T*B1 // sparse.sparsemtv(x, b1, ref c1, _params); vv = 0; for(i=0; i<=npoints-1; i++) { vv = vv+b1[i]; } for(j=0; j<=nvars-1; j++) { solver.ax[j,k] = c1[j]-vv*means[j]; } } } evd.eigsubspaceoocstop(solver, ref s2, ref v, rep, _params); if( npoints!=1 ) { for(i=0; i<=nneeded-1; i++) { s2[i] = s2[i]/(npoints-1); } } } } public class bdss { public class cvreport : apobject { public double relclserror; public double avgce; public double rmserror; public double avgerror; public double avgrelerror; public cvreport() { init(); } public override void init() { } public override alglib.apobject make_copy() { cvreport _result = new cvreport(); _result.relclserror = relclserror; _result.avgce = avgce; _result.rmserror = rmserror; _result.avgerror = avgerror; _result.avgrelerror = avgrelerror; return _result; } }; /************************************************************************* This set of routines (DSErrAllocate, DSErrAccumulate, DSErrFinish) calculates different error functions (classification error, cross-entropy, rms, avg, avg.rel errors). 1. DSErrAllocate prepares buffer. 2. DSErrAccumulate accumulates individual errors: * Y contains predicted output (posterior probabilities for classification) * DesiredY contains desired output (class number for classification) 3. DSErrFinish outputs results: * Buf[0] contains relative classification error (zero for regression tasks) * Buf[1] contains avg. cross-entropy (zero for regression tasks) * Buf[2] contains rms error (regression, classification) * Buf[3] contains average error (regression, classification) * Buf[4] contains average relative error (regression, classification) NOTES(1): "NClasses>0" means that we have classification task. "NClasses<0" means regression task with -NClasses real outputs. NOTES(2): rms. avg, avg.rel errors for classification tasks are interpreted as errors in posterior probabilities with respect to probabilities given by training/test set. -- ALGLIB -- Copyright 11.01.2009 by Bochkanov Sergey *************************************************************************/ public static void dserrallocate(int nclasses, ref double[] buf, alglib.xparams _params) { buf = new double[0]; buf = new double[7+1]; buf[0] = 0; buf[1] = 0; buf[2] = 0; buf[3] = 0; buf[4] = 0; buf[5] = nclasses; buf[6] = 0; buf[7] = 0; } /************************************************************************* See DSErrAllocate for comments on this routine. -- ALGLIB -- Copyright 11.01.2009 by Bochkanov Sergey *************************************************************************/ public static void dserraccumulate(ref double[] buf, double[] y, double[] desiredy, alglib.xparams _params) { int nclasses = 0; int nout = 0; int offs = 0; int mmax = 0; int rmax = 0; int j = 0; double v = 0; double ev = 0; offs = 5; nclasses = (int)Math.Round(buf[offs]); if( nclasses>0 ) { // // Classification // rmax = (int)Math.Round(desiredy[0]); mmax = 0; for(j=1; j<=nclasses-1; j++) { if( (double)(y[j])>(double)(y[mmax]) ) { mmax = j; } } if( mmax!=rmax ) { buf[0] = buf[0]+1; } if( (double)(y[rmax])>(double)(0) ) { buf[1] = buf[1]-Math.Log(y[rmax]); } else { buf[1] = buf[1]+Math.Log(math.maxrealnumber); } for(j=0; j<=nclasses-1; j++) { v = y[j]; if( j==rmax ) { ev = 1; } else { ev = 0; } buf[2] = buf[2]+math.sqr(v-ev); buf[3] = buf[3]+Math.Abs(v-ev); if( (double)(ev)!=(double)(0) ) { buf[4] = buf[4]+Math.Abs((v-ev)/ev); buf[offs+2] = buf[offs+2]+1; } } buf[offs+1] = buf[offs+1]+1; } else { // // Regression // nout = -nclasses; rmax = 0; for(j=1; j<=nout-1; j++) { if( (double)(desiredy[j])>(double)(desiredy[rmax]) ) { rmax = j; } } mmax = 0; for(j=1; j<=nout-1; j++) { if( (double)(y[j])>(double)(y[mmax]) ) { mmax = j; } } if( mmax!=rmax ) { buf[0] = buf[0]+1; } for(j=0; j<=nout-1; j++) { v = y[j]; ev = desiredy[j]; buf[2] = buf[2]+math.sqr(v-ev); buf[3] = buf[3]+Math.Abs(v-ev); if( (double)(ev)!=(double)(0) ) { buf[4] = buf[4]+Math.Abs((v-ev)/ev); buf[offs+2] = buf[offs+2]+1; } } buf[offs+1] = buf[offs+1]+1; } } /************************************************************************* See DSErrAllocate for comments on this routine. -- ALGLIB -- Copyright 11.01.2009 by Bochkanov Sergey *************************************************************************/ public static void dserrfinish(ref double[] buf, alglib.xparams _params) { int nout = 0; int offs = 0; offs = 5; nout = Math.Abs((int)Math.Round(buf[offs])); if( (double)(buf[offs+1])!=(double)(0) ) { buf[0] = buf[0]/buf[offs+1]; buf[1] = buf[1]/buf[offs+1]; buf[2] = Math.Sqrt(buf[2]/(nout*buf[offs+1])); buf[3] = buf[3]/(nout*buf[offs+1]); } if( (double)(buf[offs+2])!=(double)(0) ) { buf[4] = buf[4]/buf[offs+2]; } } /************************************************************************* -- ALGLIB -- Copyright 19.05.2008 by Bochkanov Sergey *************************************************************************/ public static void dsnormalize(ref double[,] xy, int npoints, int nvars, ref int info, ref double[] means, ref double[] sigmas, alglib.xparams _params) { int i = 0; int j = 0; double[] tmp = new double[0]; double mean = 0; double variance = 0; double skewness = 0; double kurtosis = 0; int i_ = 0; info = 0; means = new double[0]; sigmas = new double[0]; // // Test parameters // if( npoints<=0 || nvars<1 ) { info = -1; return; } info = 1; // // Standartization // means = new double[nvars-1+1]; sigmas = new double[nvars-1+1]; tmp = new double[npoints-1+1]; for(j=0; j<=nvars-1; j++) { for(i_=0; i_<=npoints-1;i_++) { tmp[i_] = xy[i_,j]; } basestat.samplemoments(tmp, npoints, ref mean, ref variance, ref skewness, ref kurtosis, _params); means[j] = mean; sigmas[j] = Math.Sqrt(variance); if( (double)(sigmas[j])==(double)(0) ) { sigmas[j] = 1; } for(i=0; i<=npoints-1; i++) { xy[i,j] = (xy[i,j]-means[j])/sigmas[j]; } } } /************************************************************************* -- ALGLIB -- Copyright 19.05.2008 by Bochkanov Sergey *************************************************************************/ public static void dsnormalizec(double[,] xy, int npoints, int nvars, ref int info, ref double[] means, ref double[] sigmas, alglib.xparams _params) { int j = 0; double[] tmp = new double[0]; double mean = 0; double variance = 0; double skewness = 0; double kurtosis = 0; int i_ = 0; info = 0; means = new double[0]; sigmas = new double[0]; // // Test parameters // if( npoints<=0 || nvars<1 ) { info = -1; return; } info = 1; // // Standartization // means = new double[nvars-1+1]; sigmas = new double[nvars-1+1]; tmp = new double[npoints-1+1]; for(j=0; j<=nvars-1; j++) { for(i_=0; i_<=npoints-1;i_++) { tmp[i_] = xy[i_,j]; } basestat.samplemoments(tmp, npoints, ref mean, ref variance, ref skewness, ref kurtosis, _params); means[j] = mean; sigmas[j] = Math.Sqrt(variance); if( (double)(sigmas[j])==(double)(0) ) { sigmas[j] = 1; } } } /************************************************************************* -- ALGLIB -- Copyright 19.05.2008 by Bochkanov Sergey *************************************************************************/ public static double dsgetmeanmindistance(double[,] xy, int npoints, int nvars, alglib.xparams _params) { double result = 0; int i = 0; int j = 0; double[] tmp = new double[0]; double[] tmp2 = new double[0]; double v = 0; int i_ = 0; // // Test parameters // if( npoints<=0 || nvars<1 ) { result = 0; return result; } // // Process // tmp = new double[npoints-1+1]; for(i=0; i<=npoints-1; i++) { tmp[i] = math.maxrealnumber; } tmp2 = new double[nvars-1+1]; for(i=0; i<=npoints-1; i++) { for(j=i+1; j<=npoints-1; j++) { for(i_=0; i_<=nvars-1;i_++) { tmp2[i_] = xy[i,i_]; } for(i_=0; i_<=nvars-1;i_++) { tmp2[i_] = tmp2[i_] - xy[j,i_]; } v = 0.0; for(i_=0; i_<=nvars-1;i_++) { v += tmp2[i_]*tmp2[i_]; } v = Math.Sqrt(v); tmp[i] = Math.Min(tmp[i], v); tmp[j] = Math.Min(tmp[j], v); } } result = 0; for(i=0; i<=npoints-1; i++) { result = result+tmp[i]/npoints; } return result; } /************************************************************************* -- ALGLIB -- Copyright 19.05.2008 by Bochkanov Sergey *************************************************************************/ public static void dstie(ref double[] a, int n, ref int[] ties, ref int tiecount, ref int[] p1, ref int[] p2, alglib.xparams _params) { int i = 0; int k = 0; int[] tmp = new int[0]; ties = new int[0]; tiecount = 0; p1 = new int[0]; p2 = new int[0]; // // Special case // if( n<=0 ) { tiecount = 0; return; } // // Sort A // tsort.tagsort(ref a, n, ref p1, ref p2, _params); // // Process ties // tiecount = 1; for(i=1; i<=n-1; i++) { if( (double)(a[i])!=(double)(a[i-1]) ) { tiecount = tiecount+1; } } ties = new int[tiecount+1]; ties[0] = 0; k = 1; for(i=1; i<=n-1; i++) { if( (double)(a[i])!=(double)(a[i-1]) ) { ties[k] = i; k = k+1; } } ties[tiecount] = n; } /************************************************************************* -- ALGLIB -- Copyright 11.12.2008 by Bochkanov Sergey *************************************************************************/ public static void dstiefasti(ref double[] a, ref int[] b, int n, ref int[] ties, ref int tiecount, ref double[] bufr, ref int[] bufi, alglib.xparams _params) { int i = 0; int k = 0; int[] tmp = new int[0]; tiecount = 0; // // Special case // if( n<=0 ) { tiecount = 0; return; } // // Sort A // tsort.tagsortfasti(ref a, ref b, ref bufr, ref bufi, n, _params); // // Process ties // ties[0] = 0; k = 1; for(i=1; i<=n-1; i++) { if( (double)(a[i])!=(double)(a[i-1]) ) { ties[k] = i; k = k+1; } } ties[k] = n; tiecount = k; } /************************************************************************* Optimal binary classification Algorithms finds optimal (=with minimal cross-entropy) binary partition. Internal subroutine. INPUT PARAMETERS: A - array[0..N-1], variable C - array[0..N-1], class numbers (0 or 1). N - array size OUTPUT PARAMETERS: Info - completetion code: * -3, all values of A[] are same (partition is impossible) * -2, one of C[] is incorrect (<0, >1) * -1, incorrect pararemets were passed (N<=0). * 1, OK Threshold- partiton boundary. Left part contains values which are strictly less than Threshold. Right part contains values which are greater than or equal to Threshold. PAL, PBL- probabilities P(0|v=Threshold) and P(1|v>=Threshold) CVE - cross-validation estimate of cross-entropy -- ALGLIB -- Copyright 22.05.2008 by Bochkanov Sergey *************************************************************************/ public static void dsoptimalsplit2(double[] a, int[] c, int n, ref int info, ref double threshold, ref double pal, ref double pbl, ref double par, ref double pbr, ref double cve, alglib.xparams _params) { int i = 0; int t = 0; double s = 0; int[] ties = new int[0]; int tiecount = 0; int[] p1 = new int[0]; int[] p2 = new int[0]; int k = 0; int koptimal = 0; double pak = 0; double pbk = 0; double cvoptimal = 0; double cv = 0; a = (double[])a.Clone(); c = (int[])c.Clone(); info = 0; threshold = 0; pal = 0; pbl = 0; par = 0; pbr = 0; cve = 0; // // Test for errors in inputs // if( n<=0 ) { info = -1; return; } for(i=0; i<=n-1; i++) { if( c[i]!=0 && c[i]!=1 ) { info = -2; return; } } info = 1; // // Tie // dstie(ref a, n, ref ties, ref tiecount, ref p1, ref p2, _params); for(i=0; i<=n-1; i++) { if( p2[i]!=i ) { t = c[i]; c[i] = c[p2[i]]; c[p2[i]] = t; } } // // Special case: number of ties is 1. // // NOTE: we assume that P[i,j] equals to 0 or 1, // intermediate values are not allowed. // if( tiecount==1 ) { info = -3; return; } // // General case, number of ties > 1 // // NOTE: we assume that P[i,j] equals to 0 or 1, // intermediate values are not allowed. // pal = 0; pbl = 0; par = 0; pbr = 0; for(i=0; i<=n-1; i++) { if( c[i]==0 ) { par = par+1; } if( c[i]==1 ) { pbr = pbr+1; } } koptimal = -1; cvoptimal = math.maxrealnumber; for(k=0; k<=tiecount-2; k++) { // // first, obtain information about K-th tie which is // moved from R-part to L-part // pak = 0; pbk = 0; for(i=ties[k]; i<=ties[k+1]-1; i++) { if( c[i]==0 ) { pak = pak+1; } if( c[i]==1 ) { pbk = pbk+1; } } // // Calculate cross-validation CE // cv = 0; cv = cv-xlny(pal+pak, (pal+pak)/(pal+pak+pbl+pbk+1), _params); cv = cv-xlny(pbl+pbk, (pbl+pbk)/(pal+pak+1+pbl+pbk), _params); cv = cv-xlny(par-pak, (par-pak)/(par-pak+pbr-pbk+1), _params); cv = cv-xlny(pbr-pbk, (pbr-pbk)/(par-pak+1+pbr-pbk), _params); // // Compare with best // if( (double)(cv)<(double)(cvoptimal) ) { cvoptimal = cv; koptimal = k; } // // update // pal = pal+pak; pbl = pbl+pbk; par = par-pak; pbr = pbr-pbk; } cve = cvoptimal; threshold = 0.5*(a[ties[koptimal]]+a[ties[koptimal+1]]); pal = 0; pbl = 0; par = 0; pbr = 0; for(i=0; i<=n-1; i++) { if( (double)(a[i])<(double)(threshold) ) { if( c[i]==0 ) { pal = pal+1; } else { pbl = pbl+1; } } else { if( c[i]==0 ) { par = par+1; } else { pbr = pbr+1; } } } s = pal+pbl; pal = pal/s; pbl = pbl/s; s = par+pbr; par = par/s; pbr = pbr/s; } /************************************************************************* Optimal partition, internal subroutine. Fast version. Accepts: A array[0..N-1] array of attributes array[0..N-1] C array[0..N-1] array of class labels TiesBuf array[0..N] temporaries (ties) CntBuf array[0..2*NC-1] temporaries (counts) Alpha centering factor (0<=alpha<=1, recommended value - 0.05) BufR array[0..N-1] temporaries BufI array[0..N-1] temporaries Output: Info error code (">0"=OK, "<0"=bad) RMS training set RMS error CVRMS leave-one-out RMS error Note: content of all arrays is changed by subroutine; it doesn't allocate temporaries. -- ALGLIB -- Copyright 11.12.2008 by Bochkanov Sergey *************************************************************************/ public static void dsoptimalsplit2fast(ref double[] a, ref int[] c, ref int[] tiesbuf, ref int[] cntbuf, ref double[] bufr, ref int[] bufi, int n, int nc, double alpha, ref int info, ref double threshold, ref double rms, ref double cvrms, alglib.xparams _params) { int i = 0; int k = 0; int cl = 0; int tiecount = 0; double cbest = 0; double cc = 0; int koptimal = 0; int sl = 0; int sr = 0; double v = 0; double w = 0; double x = 0; info = 0; threshold = 0; rms = 0; cvrms = 0; // // Test for errors in inputs // if( n<=0 || nc<2 ) { info = -1; return; } for(i=0; i<=n-1; i++) { if( c[i]<0 || c[i]>=nc ) { info = -2; return; } } info = 1; // // Tie // dstiefasti(ref a, ref c, n, ref tiesbuf, ref tiecount, ref bufr, ref bufi, _params); // // Special case: number of ties is 1. // if( tiecount==1 ) { info = -3; return; } // // General case, number of ties > 1 // for(i=0; i<=2*nc-1; i++) { cntbuf[i] = 0; } for(i=0; i<=n-1; i++) { cntbuf[nc+c[i]] = cntbuf[nc+c[i]]+1; } koptimal = -1; threshold = a[n-1]; cbest = math.maxrealnumber; sl = 0; sr = n; for(k=0; k<=tiecount-2; k++) { // // first, move Kth tie from right to left // for(i=tiesbuf[k]; i<=tiesbuf[k+1]-1; i++) { cl = c[i]; cntbuf[cl] = cntbuf[cl]+1; cntbuf[nc+cl] = cntbuf[nc+cl]-1; } sl = sl+(tiesbuf[k+1]-tiesbuf[k]); sr = sr-(tiesbuf[k+1]-tiesbuf[k]); // // Calculate RMS error // v = 0; for(i=0; i<=nc-1; i++) { w = cntbuf[i]; v = v+w*math.sqr(w/sl-1); v = v+(sl-w)*math.sqr(w/sl); w = cntbuf[nc+i]; v = v+w*math.sqr(w/sr-1); v = v+(sr-w)*math.sqr(w/sr); } v = Math.Sqrt(v/(nc*n)); // // Compare with best // x = (double)(2*sl)/(double)(sl+sr)-1; cc = v*(1-alpha+alpha*math.sqr(x)); if( (double)(cc)<(double)(cbest) ) { // // store split // rms = v; koptimal = k; cbest = cc; // // calculate CVRMS error // cvrms = 0; for(i=0; i<=nc-1; i++) { if( sl>1 ) { w = cntbuf[i]; cvrms = cvrms+w*math.sqr((w-1)/(sl-1)-1); cvrms = cvrms+(sl-w)*math.sqr(w/(sl-1)); } else { w = cntbuf[i]; cvrms = cvrms+w*math.sqr((double)1/(double)nc-1); cvrms = cvrms+(sl-w)*math.sqr((double)1/(double)nc); } if( sr>1 ) { w = cntbuf[nc+i]; cvrms = cvrms+w*math.sqr((w-1)/(sr-1)-1); cvrms = cvrms+(sr-w)*math.sqr(w/(sr-1)); } else { w = cntbuf[nc+i]; cvrms = cvrms+w*math.sqr((double)1/(double)nc-1); cvrms = cvrms+(sr-w)*math.sqr((double)1/(double)nc); } } cvrms = Math.Sqrt(cvrms/(nc*n)); } } // // Calculate threshold. // Code is a bit complicated because there can be such // numbers that 0.5(A+B) equals to A or B (if A-B=epsilon) // threshold = 0.5*(a[tiesbuf[koptimal]]+a[tiesbuf[koptimal+1]]); if( (double)(threshold)<=(double)(a[tiesbuf[koptimal]]) ) { threshold = a[tiesbuf[koptimal+1]]; } } /************************************************************************* Automatic non-optimal discretization, internal subroutine. -- ALGLIB -- Copyright 22.05.2008 by Bochkanov Sergey *************************************************************************/ public static void dssplitk(double[] a, int[] c, int n, int nc, int kmax, ref int info, ref double[] thresholds, ref int ni, ref double cve, alglib.xparams _params) { int i = 0; int j = 0; int j1 = 0; int k = 0; int[] ties = new int[0]; int tiecount = 0; int[] p1 = new int[0]; int[] p2 = new int[0]; int[] cnt = new int[0]; double v2 = 0; int bestk = 0; double bestcve = 0; int[] bestsizes = new int[0]; double curcve = 0; int[] cursizes = new int[0]; a = (double[])a.Clone(); c = (int[])c.Clone(); info = 0; thresholds = new double[0]; ni = 0; cve = 0; // // Test for errors in inputs // if( (n<=0 || nc<2) || kmax<2 ) { info = -1; return; } for(i=0; i<=n-1; i++) { if( c[i]<0 || c[i]>=nc ) { info = -2; return; } } info = 1; // // Tie // dstie(ref a, n, ref ties, ref tiecount, ref p1, ref p2, _params); for(i=0; i<=n-1; i++) { if( p2[i]!=i ) { k = c[i]; c[i] = c[p2[i]]; c[p2[i]] = k; } } // // Special cases // if( tiecount==1 ) { info = -3; return; } // // General case: // 0. allocate arrays // kmax = Math.Min(kmax, tiecount); bestsizes = new int[kmax-1+1]; cursizes = new int[kmax-1+1]; cnt = new int[nc-1+1]; // // General case: // 1. prepare "weak" solution (two subintervals, divided at median) // v2 = math.maxrealnumber; j = -1; for(i=1; i<=tiecount-1; i++) { if( (double)(Math.Abs(ties[i]-0.5*(n-1)))<(double)(v2) ) { v2 = Math.Abs(ties[i]-0.5*n); j = i; } } alglib.ap.assert(j>0, "DSSplitK: internal error #1!"); bestk = 2; bestsizes[0] = ties[j]; bestsizes[1] = n-j; bestcve = 0; for(i=0; i<=nc-1; i++) { cnt[i] = 0; } for(i=0; i<=j-1; i++) { tieaddc(c, ties, i, nc, ref cnt, _params); } bestcve = bestcve+getcv(cnt, nc, _params); for(i=0; i<=nc-1; i++) { cnt[i] = 0; } for(i=j; i<=tiecount-1; i++) { tieaddc(c, ties, i, nc, ref cnt, _params); } bestcve = bestcve+getcv(cnt, nc, _params); // // General case: // 2. Use greedy algorithm to find sub-optimal split in O(KMax*N) time // for(k=2; k<=kmax; k++) { // // Prepare greedy K-interval split // for(i=0; i<=k-1; i++) { cursizes[i] = 0; } i = 0; j = 0; while( j<=tiecount-1 && i<=k-1 ) { // // Rule: I-th bin is empty, fill it // if( cursizes[i]==0 ) { cursizes[i] = ties[j+1]-ties[j]; j = j+1; continue; } // // Rule: (K-1-I) bins left, (K-1-I) ties left (1 tie per bin); next bin // if( tiecount-j==k-1-i ) { i = i+1; continue; } // // Rule: last bin, always place in current // if( i==k-1 ) { cursizes[i] = cursizes[i]+ties[j+1]-ties[j]; j = j+1; continue; } // // Place J-th tie in I-th bin, or leave for I+1-th bin. // if( (double)(Math.Abs(cursizes[i]+ties[j+1]-ties[j]-(double)n/(double)k))<(double)(Math.Abs(cursizes[i]-(double)n/(double)k)) ) { cursizes[i] = cursizes[i]+ties[j+1]-ties[j]; j = j+1; } else { i = i+1; } } alglib.ap.assert(cursizes[k-1]!=0 && j==tiecount, "DSSplitK: internal error #1"); // // Calculate CVE // curcve = 0; j = 0; for(i=0; i<=k-1; i++) { for(j1=0; j1<=nc-1; j1++) { cnt[j1] = 0; } for(j1=j; j1<=j+cursizes[i]-1; j1++) { cnt[c[j1]] = cnt[c[j1]]+1; } curcve = curcve+getcv(cnt, nc, _params); j = j+cursizes[i]; } // // Choose best variant // if( (double)(curcve)<(double)(bestcve) ) { for(i=0; i<=k-1; i++) { bestsizes[i] = cursizes[i]; } bestcve = curcve; bestk = k; } } // // Transform from sizes to thresholds // cve = bestcve; ni = bestk; thresholds = new double[ni-2+1]; j = bestsizes[0]; for(i=1; i<=bestk-1; i++) { thresholds[i-1] = 0.5*(a[j-1]+a[j]); j = j+bestsizes[i]; } } /************************************************************************* Automatic optimal discretization, internal subroutine. -- ALGLIB -- Copyright 22.05.2008 by Bochkanov Sergey *************************************************************************/ public static void dsoptimalsplitk(double[] a, int[] c, int n, int nc, int kmax, ref int info, ref double[] thresholds, ref int ni, ref double cve, alglib.xparams _params) { int i = 0; int j = 0; int s = 0; int jl = 0; int jr = 0; double v2 = 0; int[] ties = new int[0]; int tiecount = 0; int[] p1 = new int[0]; int[] p2 = new int[0]; double cvtemp = 0; int[] cnt = new int[0]; int[] cnt2 = new int[0]; double[,] cv = new double[0,0]; int[,] splits = new int[0,0]; int k = 0; int koptimal = 0; double cvoptimal = 0; a = (double[])a.Clone(); c = (int[])c.Clone(); info = 0; thresholds = new double[0]; ni = 0; cve = 0; // // Test for errors in inputs // if( (n<=0 || nc<2) || kmax<2 ) { info = -1; return; } for(i=0; i<=n-1; i++) { if( c[i]<0 || c[i]>=nc ) { info = -2; return; } } info = 1; // // Tie // dstie(ref a, n, ref ties, ref tiecount, ref p1, ref p2, _params); for(i=0; i<=n-1; i++) { if( p2[i]!=i ) { k = c[i]; c[i] = c[p2[i]]; c[p2[i]] = k; } } // // Special cases // if( tiecount==1 ) { info = -3; return; } // // General case // Use dynamic programming to find best split in O(KMax*NC*TieCount^2) time // kmax = Math.Min(kmax, tiecount); cv = new double[kmax-1+1, tiecount-1+1]; splits = new int[kmax-1+1, tiecount-1+1]; cnt = new int[nc-1+1]; cnt2 = new int[nc-1+1]; for(j=0; j<=nc-1; j++) { cnt[j] = 0; } for(j=0; j<=tiecount-1; j++) { tieaddc(c, ties, j, nc, ref cnt, _params); splits[0,j] = 0; cv[0,j] = getcv(cnt, nc, _params); } for(k=1; k<=kmax-1; k++) { for(j=0; j<=nc-1; j++) { cnt[j] = 0; } // // Subtask size J in [K..TieCount-1]: // optimal K-splitting on ties from 0-th to J-th. // for(j=k; j<=tiecount-1; j++) { // // Update Cnt - let it contain classes of ties from K-th to J-th // tieaddc(c, ties, j, nc, ref cnt, _params); // // Search for optimal split point S in [K..J] // for(i=0; i<=nc-1; i++) { cnt2[i] = cnt[i]; } cv[k,j] = cv[k-1,j-1]+getcv(cnt2, nc, _params); splits[k,j] = j; for(s=k+1; s<=j; s++) { // // Update Cnt2 - let it contain classes of ties from S-th to J-th // tiesubc(c, ties, s-1, nc, ref cnt2, _params); // // Calculate CVE // cvtemp = cv[k-1,s-1]+getcv(cnt2, nc, _params); if( (double)(cvtemp)<(double)(cv[k,j]) ) { cv[k,j] = cvtemp; splits[k,j] = s; } } } } // // Choose best partition, output result // koptimal = -1; cvoptimal = math.maxrealnumber; for(k=0; k<=kmax-1; k++) { if( (double)(cv[k,tiecount-1])<(double)(cvoptimal) ) { cvoptimal = cv[k,tiecount-1]; koptimal = k; } } alglib.ap.assert(koptimal>=0, "DSOptimalSplitK: internal error #1!"); if( koptimal==0 ) { // // Special case: best partition is one big interval. // Even 2-partition is not better. // This is possible when dealing with "weak" predictor variables. // // Make binary split as close to the median as possible. // v2 = math.maxrealnumber; j = -1; for(i=1; i<=tiecount-1; i++) { if( (double)(Math.Abs(ties[i]-0.5*(n-1)))<(double)(v2) ) { v2 = Math.Abs(ties[i]-0.5*(n-1)); j = i; } } alglib.ap.assert(j>0, "DSOptimalSplitK: internal error #2!"); thresholds = new double[0+1]; thresholds[0] = 0.5*(a[ties[j-1]]+a[ties[j]]); ni = 2; cve = 0; for(i=0; i<=nc-1; i++) { cnt[i] = 0; } for(i=0; i<=j-1; i++) { tieaddc(c, ties, i, nc, ref cnt, _params); } cve = cve+getcv(cnt, nc, _params); for(i=0; i<=nc-1; i++) { cnt[i] = 0; } for(i=j; i<=tiecount-1; i++) { tieaddc(c, ties, i, nc, ref cnt, _params); } cve = cve+getcv(cnt, nc, _params); } else { // // General case: 2 or more intervals // // NOTE: we initialize both JL and JR (left and right bounds), // altough algorithm needs only JL. // thresholds = new double[koptimal-1+1]; ni = koptimal+1; cve = cv[koptimal,tiecount-1]; jl = splits[koptimal,tiecount-1]; jr = tiecount-1; for(k=koptimal; k>=1; k--) { thresholds[k-1] = 0.5*(a[ties[jl-1]]+a[ties[jl]]); jr = jl-1; jl = splits[k-1,jl-1]; } apserv.touchint(ref jr, _params); } } /************************************************************************* Internal function *************************************************************************/ private static double xlny(double x, double y, alglib.xparams _params) { double result = 0; if( (double)(x)==(double)(0) ) { result = 0; } else { result = x*Math.Log(y); } return result; } /************************************************************************* Internal function, returns number of samples of class I in Cnt[I] *************************************************************************/ private static double getcv(int[] cnt, int nc, alglib.xparams _params) { double result = 0; int i = 0; double s = 0; s = 0; for(i=0; i<=nc-1; i++) { s = s+cnt[i]; } result = 0; for(i=0; i<=nc-1; i++) { result = result-xlny(cnt[i], cnt[i]/(s+nc-1), _params); } return result; } /************************************************************************* Internal function, adds number of samples of class I in tie NTie to Cnt[I] *************************************************************************/ private static void tieaddc(int[] c, int[] ties, int ntie, int nc, ref int[] cnt, alglib.xparams _params) { int i = 0; for(i=ties[ntie]; i<=ties[ntie+1]-1; i++) { cnt[c[i]] = cnt[c[i]]+1; } } /************************************************************************* Internal function, subtracts number of samples of class I in tie NTie to Cnt[I] *************************************************************************/ private static void tiesubc(int[] c, int[] ties, int ntie, int nc, ref int[] cnt, alglib.xparams _params) { int i = 0; for(i=ties[ntie]; i<=ties[ntie+1]-1; i++) { cnt[c[i]] = cnt[c[i]]-1; } } } public class mlpbase { /************************************************************************* Model's errors: * RelCLSError - fraction of misclassified cases. * AvgCE - acerage cross-entropy * RMSError - root-mean-square error * AvgError - average error * AvgRelError - average relative error NOTE 1: RelCLSError/AvgCE are zero on regression problems. NOTE 2: on classification problems RMSError/AvgError/AvgRelError contain errors in prediction of posterior probabilities *************************************************************************/ public class modelerrors : apobject { public double relclserror; public double avgce; public double rmserror; public double avgerror; public double avgrelerror; public modelerrors() { init(); } public override void init() { } public override alglib.apobject make_copy() { modelerrors _result = new modelerrors(); _result.relclserror = relclserror; _result.avgce = avgce; _result.rmserror = rmserror; _result.avgerror = avgerror; _result.avgrelerror = avgrelerror; return _result; } }; /************************************************************************* This structure is used to store MLP error and gradient. *************************************************************************/ public class smlpgrad : apobject { public double f; public double[] g; public smlpgrad() { init(); } public override void init() { g = new double[0]; } public override alglib.apobject make_copy() { smlpgrad _result = new smlpgrad(); _result.f = f; _result.g = (double[])g.Clone(); return _result; } }; public class multilayerperceptron : apobject { public int hlnetworktype; public int hlnormtype; public int[] hllayersizes; public int[] hlconnections; public int[] hlneurons; public int[] structinfo; public double[] weights; public double[] columnmeans; public double[] columnsigmas; public double[] neurons; public double[] dfdnet; public double[] derror; public double[] x; public double[] y; public double[,] xy; public double[] xyrow; public double[] nwbuf; public int[] integerbuf; public modelerrors err; public double[] rndbuf; public alglib.smp.shared_pool buf; public alglib.smp.shared_pool gradbuf; public double[,] dummydxy; public sparse.sparsematrix dummysxy; public int[] dummyidx; public alglib.smp.shared_pool dummypool; public multilayerperceptron() { init(); } public override void init() { hllayersizes = new int[0]; hlconnections = new int[0]; hlneurons = new int[0]; structinfo = new int[0]; weights = new double[0]; columnmeans = new double[0]; columnsigmas = new double[0]; neurons = new double[0]; dfdnet = new double[0]; derror = new double[0]; x = new double[0]; y = new double[0]; xy = new double[0,0]; xyrow = new double[0]; nwbuf = new double[0]; integerbuf = new int[0]; err = new modelerrors(); rndbuf = new double[0]; buf = new alglib.smp.shared_pool(); gradbuf = new alglib.smp.shared_pool(); dummydxy = new double[0,0]; dummysxy = new sparse.sparsematrix(); dummyidx = new int[0]; dummypool = new alglib.smp.shared_pool(); } public override alglib.apobject make_copy() { multilayerperceptron _result = new multilayerperceptron(); _result.hlnetworktype = hlnetworktype; _result.hlnormtype = hlnormtype; _result.hllayersizes = (int[])hllayersizes.Clone(); _result.hlconnections = (int[])hlconnections.Clone(); _result.hlneurons = (int[])hlneurons.Clone(); _result.structinfo = (int[])structinfo.Clone(); _result.weights = (double[])weights.Clone(); _result.columnmeans = (double[])columnmeans.Clone(); _result.columnsigmas = (double[])columnsigmas.Clone(); _result.neurons = (double[])neurons.Clone(); _result.dfdnet = (double[])dfdnet.Clone(); _result.derror = (double[])derror.Clone(); _result.x = (double[])x.Clone(); _result.y = (double[])y.Clone(); _result.xy = (double[,])xy.Clone(); _result.xyrow = (double[])xyrow.Clone(); _result.nwbuf = (double[])nwbuf.Clone(); _result.integerbuf = (int[])integerbuf.Clone(); _result.err = (modelerrors)err.make_copy(); _result.rndbuf = (double[])rndbuf.Clone(); _result.buf = (alglib.smp.shared_pool)buf.make_copy(); _result.gradbuf = (alglib.smp.shared_pool)gradbuf.make_copy(); _result.dummydxy = (double[,])dummydxy.Clone(); _result.dummysxy = (sparse.sparsematrix)dummysxy.make_copy(); _result.dummyidx = (int[])dummyidx.Clone(); _result.dummypool = (alglib.smp.shared_pool)dummypool.make_copy(); return _result; } }; public const int mlpvnum = 7; public const int mlpfirstversion = 0; public const int nfieldwidth = 4; public const int hlconnfieldwidth = 5; public const int hlnfieldwidth = 4; public const int gradbasecasecost = 50000; public const int microbatchsize = 64; /************************************************************************* This function returns number of weights updates which is required for gradient calculation problem to be splitted. *************************************************************************/ public static int mlpgradsplitcost(alglib.xparams _params) { int result = 0; result = gradbasecasecost; return result; } /************************************************************************* This function returns number of elements in subset of dataset which is required for gradient calculation problem to be splitted. *************************************************************************/ public static int mlpgradsplitsize(alglib.xparams _params) { int result = 0; result = microbatchsize; return result; } /************************************************************************* Creates neural network with NIn inputs, NOut outputs, without hidden layers, with linear output layer. Network weights are filled with small random values. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreate0(int nin, int nout, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; layerscount = 1+3; // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(-5, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, false, network, _params); fillhighlevelinformation(network, nin, 0, 0, nout, false, true, _params); } /************************************************************************* Same as MLPCreate0, but with one hidden layer (NHid neurons) with non-linear activation function. Output layer is linear. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreate1(int nin, int nhid, int nout, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; layerscount = 1+3+3; // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(-5, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, false, network, _params); fillhighlevelinformation(network, nin, nhid, 0, nout, false, true, _params); } /************************************************************************* Same as MLPCreate0, but with two hidden layers (NHid1 and NHid2 neurons) with non-linear activation function. Output layer is linear. $ALL -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreate2(int nin, int nhid1, int nhid2, int nout, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; layerscount = 1+3+3+3; // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid2, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(-5, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, false, network, _params); fillhighlevelinformation(network, nin, nhid1, nhid2, nout, false, true, _params); } /************************************************************************* Creates neural network with NIn inputs, NOut outputs, without hidden layers with non-linear output layer. Network weights are filled with small random values. Activation function of the output layer takes values: (B, +INF), if D>=0 or (-INF, B), if D<0. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreateb0(int nin, int nout, double b, double d, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; int i = 0; layerscount = 1+3; if( (double)(d)>=(double)(0) ) { d = 1; } else { d = -1; } // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(3, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, false, network, _params); fillhighlevelinformation(network, nin, 0, 0, nout, false, false, _params); // // Turn on ouputs shift/scaling. // for(i=nin; i<=nin+nout-1; i++) { network.columnmeans[i] = b; network.columnsigmas[i] = d; } } /************************************************************************* Same as MLPCreateB0 but with non-linear hidden layer. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreateb1(int nin, int nhid, int nout, double b, double d, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; int i = 0; layerscount = 1+3+3; if( (double)(d)>=(double)(0) ) { d = 1; } else { d = -1; } // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(3, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, false, network, _params); fillhighlevelinformation(network, nin, nhid, 0, nout, false, false, _params); // // Turn on ouputs shift/scaling. // for(i=nin; i<=nin+nout-1; i++) { network.columnmeans[i] = b; network.columnsigmas[i] = d; } } /************************************************************************* Same as MLPCreateB0 but with two non-linear hidden layers. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreateb2(int nin, int nhid1, int nhid2, int nout, double b, double d, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; int i = 0; layerscount = 1+3+3+3; if( (double)(d)>=(double)(0) ) { d = 1; } else { d = -1; } // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid2, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(3, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, false, network, _params); fillhighlevelinformation(network, nin, nhid1, nhid2, nout, false, false, _params); // // Turn on ouputs shift/scaling. // for(i=nin; i<=nin+nout-1; i++) { network.columnmeans[i] = b; network.columnsigmas[i] = d; } } /************************************************************************* Creates neural network with NIn inputs, NOut outputs, without hidden layers with non-linear output layer. Network weights are filled with small random values. Activation function of the output layer takes values [A,B]. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreater0(int nin, int nout, double a, double b, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; int i = 0; layerscount = 1+3; // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, false, network, _params); fillhighlevelinformation(network, nin, 0, 0, nout, false, false, _params); // // Turn on outputs shift/scaling. // for(i=nin; i<=nin+nout-1; i++) { network.columnmeans[i] = 0.5*(a+b); network.columnsigmas[i] = 0.5*(a-b); } } /************************************************************************* Same as MLPCreateR0, but with non-linear hidden layer. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreater1(int nin, int nhid, int nout, double a, double b, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; int i = 0; layerscount = 1+3+3; // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, false, network, _params); fillhighlevelinformation(network, nin, nhid, 0, nout, false, false, _params); // // Turn on outputs shift/scaling. // for(i=nin; i<=nin+nout-1; i++) { network.columnmeans[i] = 0.5*(a+b); network.columnsigmas[i] = 0.5*(a-b); } } /************************************************************************* Same as MLPCreateR0, but with two non-linear hidden layers. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpcreater2(int nin, int nhid1, int nhid2, int nout, double a, double b, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; int i = 0; layerscount = 1+3+3+3; // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid2, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, false, network, _params); fillhighlevelinformation(network, nin, nhid1, nhid2, nout, false, false, _params); // // Turn on outputs shift/scaling. // for(i=nin; i<=nin+nout-1; i++) { network.columnmeans[i] = 0.5*(a+b); network.columnsigmas[i] = 0.5*(a-b); } } /************************************************************************* Creates classifier network with NIn inputs and NOut possible classes. Network contains no hidden layers and linear output layer with SOFTMAX- normalization (so outputs sums up to 1.0 and converge to posterior probabilities). -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatec0(int nin, int nout, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; alglib.ap.assert(nout>=2, "MLPCreateC0: NOut<2!"); layerscount = 1+2+1; // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout-1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addzerolayer(ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, true, network, _params); fillhighlevelinformation(network, nin, 0, 0, nout, true, true, _params); } /************************************************************************* Same as MLPCreateC0, but with one non-linear hidden layer. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatec1(int nin, int nhid, int nout, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; alglib.ap.assert(nout>=2, "MLPCreateC1: NOut<2!"); layerscount = 1+3+2+1; // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout-1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addzerolayer(ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, true, network, _params); fillhighlevelinformation(network, nin, nhid, 0, nout, true, true, _params); } /************************************************************************* Same as MLPCreateC0, but with two non-linear hidden layers. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatec2(int nin, int nhid1, int nhid2, int nout, multilayerperceptron network, alglib.xparams _params) { int[] lsizes = new int[0]; int[] ltypes = new int[0]; int[] lconnfirst = new int[0]; int[] lconnlast = new int[0]; int layerscount = 0; int lastproc = 0; alglib.ap.assert(nout>=2, "MLPCreateC2: NOut<2!"); layerscount = 1+3+3+2+1; // // Allocate arrays // lsizes = new int[layerscount-1+1]; ltypes = new int[layerscount-1+1]; lconnfirst = new int[layerscount-1+1]; lconnlast = new int[layerscount-1+1]; // // Layers // addinputlayer(nin, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nhid2, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addactivationlayer(1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addbiasedsummatorlayer(nout-1, ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); addzerolayer(ref lsizes, ref ltypes, ref lconnfirst, ref lconnlast, ref lastproc, _params); // // Create // mlpcreate(nin, nout, lsizes, ltypes, lconnfirst, lconnlast, layerscount, true, network, _params); fillhighlevelinformation(network, nin, nhid1, nhid2, nout, true, true, _params); } /************************************************************************* Copying of neural network INPUT PARAMETERS: Network1 - original OUTPUT PARAMETERS: Network2 - copy -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcopy(multilayerperceptron network1, multilayerperceptron network2, alglib.xparams _params) { mlpcopyshared(network1, network2, _params); } /************************************************************************* Copying of neural network (second parameter is passed as shared object). INPUT PARAMETERS: Network1 - original OUTPUT PARAMETERS: Network2 - copy -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpcopyshared(multilayerperceptron network1, multilayerperceptron network2, alglib.xparams _params) { int wcount = 0; int i = 0; hpccores.mlpbuffers buf = new hpccores.mlpbuffers(); smlpgrad sgrad = new smlpgrad(); // // Copy scalar and array fields // network2.hlnetworktype = network1.hlnetworktype; network2.hlnormtype = network1.hlnormtype; apserv.copyintegerarray(network1.hllayersizes, ref network2.hllayersizes, _params); apserv.copyintegerarray(network1.hlconnections, ref network2.hlconnections, _params); apserv.copyintegerarray(network1.hlneurons, ref network2.hlneurons, _params); apserv.copyintegerarray(network1.structinfo, ref network2.structinfo, _params); apserv.copyrealarray(network1.weights, ref network2.weights, _params); apserv.copyrealarray(network1.columnmeans, ref network2.columnmeans, _params); apserv.copyrealarray(network1.columnsigmas, ref network2.columnsigmas, _params); apserv.copyrealarray(network1.neurons, ref network2.neurons, _params); apserv.copyrealarray(network1.dfdnet, ref network2.dfdnet, _params); apserv.copyrealarray(network1.derror, ref network2.derror, _params); apserv.copyrealarray(network1.x, ref network2.x, _params); apserv.copyrealarray(network1.y, ref network2.y, _params); apserv.copyrealarray(network1.nwbuf, ref network2.nwbuf, _params); apserv.copyintegerarray(network1.integerbuf, ref network2.integerbuf, _params); // // copy buffers // wcount = mlpgetweightscount(network1, _params); alglib.smp.ae_shared_pool_set_seed(network2.buf, buf); sgrad.g = new double[wcount]; sgrad.f = 0.0; for(i=0; i<=wcount-1; i++) { sgrad.g[i] = 0.0; } alglib.smp.ae_shared_pool_set_seed(network2.gradbuf, sgrad); } /************************************************************************* This function compares architectures of neural networks. Only geometries are compared, weights and other parameters are not tested. -- ALGLIB -- Copyright 20.06.2013 by Bochkanov Sergey *************************************************************************/ public static bool mlpsamearchitecture(multilayerperceptron network1, multilayerperceptron network2, alglib.xparams _params) { bool result = new bool(); int i = 0; int ninfo = 0; alglib.ap.assert(alglib.ap.len(network1.structinfo)>0 && alglib.ap.len(network1.structinfo)>=network1.structinfo[0], "MLPSameArchitecture: Network1 is uninitialized"); alglib.ap.assert(alglib.ap.len(network2.structinfo)>0 && alglib.ap.len(network2.structinfo)>=network2.structinfo[0], "MLPSameArchitecture: Network2 is uninitialized"); result = false; if( network1.structinfo[0]!=network2.structinfo[0] ) { return result; } ninfo = network1.structinfo[0]; for(i=0; i<=ninfo-1; i++) { if( network1.structinfo[i]!=network2.structinfo[i] ) { return result; } } result = true; return result; } /************************************************************************* This function copies tunable parameters (weights/means/sigmas) from one network to another with same architecture. It performs some rudimentary checks that architectures are same, and throws exception if check fails. It is intended for fast copying of states between two network which are known to have same geometry. INPUT PARAMETERS: Network1 - source, must be correctly initialized Network2 - target, must have same architecture OUTPUT PARAMETERS: Network2 - network state is copied from source to target -- ALGLIB -- Copyright 20.06.2013 by Bochkanov Sergey *************************************************************************/ public static void mlpcopytunableparameters(multilayerperceptron network1, multilayerperceptron network2, alglib.xparams _params) { int i = 0; int ninfo = 0; int nin = 0; int nout = 0; int wcount = 0; alglib.ap.assert(alglib.ap.len(network1.structinfo)>0 && alglib.ap.len(network1.structinfo)>=network1.structinfo[0], "MLPCopyTunableParameters: Network1 is uninitialized"); alglib.ap.assert(alglib.ap.len(network2.structinfo)>0 && alglib.ap.len(network2.structinfo)>=network2.structinfo[0], "MLPCopyTunableParameters: Network2 is uninitialized"); alglib.ap.assert(network1.structinfo[0]==network2.structinfo[0], "MLPCopyTunableParameters: Network1 geometry differs from that of Network2"); ninfo = network1.structinfo[0]; for(i=0; i<=ninfo-1; i++) { alglib.ap.assert(network1.structinfo[i]==network2.structinfo[i], "MLPCopyTunableParameters: Network1 geometry differs from that of Network2"); } mlpproperties(network1, ref nin, ref nout, ref wcount, _params); for(i=0; i<=wcount-1; i++) { network2.weights[i] = network1.weights[i]; } if( mlpissoftmax(network1, _params) ) { for(i=0; i<=nin-1; i++) { network2.columnmeans[i] = network1.columnmeans[i]; network2.columnsigmas[i] = network1.columnsigmas[i]; } } else { for(i=0; i<=nin+nout-1; i++) { network2.columnmeans[i] = network1.columnmeans[i]; network2.columnsigmas[i] = network1.columnsigmas[i]; } } } /************************************************************************* This function exports tunable parameters (weights/means/sigmas) from network to contiguous array. Nothing is guaranteed about array format, the only thing you can count for is that MLPImportTunableParameters() will be able to parse it. It is intended for fast copying of states between network and backup array INPUT PARAMETERS: Network - source, must be correctly initialized P - array to use. If its size is enough to store data, it is reused. OUTPUT PARAMETERS: P - array which stores network parameters, resized if needed PCount - number of parameters stored in array. -- ALGLIB -- Copyright 20.06.2013 by Bochkanov Sergey *************************************************************************/ public static void mlpexporttunableparameters(multilayerperceptron network, ref double[] p, ref int pcount, alglib.xparams _params) { int i = 0; int k = 0; int nin = 0; int nout = 0; int wcount = 0; pcount = 0; alglib.ap.assert(alglib.ap.len(network.structinfo)>0 && alglib.ap.len(network.structinfo)>=network.structinfo[0], "MLPExportTunableParameters: Network is uninitialized"); mlpproperties(network, ref nin, ref nout, ref wcount, _params); if( mlpissoftmax(network, _params) ) { pcount = wcount+2*nin; apserv.rvectorsetlengthatleast(ref p, pcount, _params); k = 0; for(i=0; i<=wcount-1; i++) { p[k] = network.weights[i]; k = k+1; } for(i=0; i<=nin-1; i++) { p[k] = network.columnmeans[i]; k = k+1; p[k] = network.columnsigmas[i]; k = k+1; } } else { pcount = wcount+2*(nin+nout); apserv.rvectorsetlengthatleast(ref p, pcount, _params); k = 0; for(i=0; i<=wcount-1; i++) { p[k] = network.weights[i]; k = k+1; } for(i=0; i<=nin+nout-1; i++) { p[k] = network.columnmeans[i]; k = k+1; p[k] = network.columnsigmas[i]; k = k+1; } } } /************************************************************************* This function imports tunable parameters (weights/means/sigmas) which were exported by MLPExportTunableParameters(). It is intended for fast copying of states between network and backup array INPUT PARAMETERS: Network - target: * must be correctly initialized * must have same geometry as network used to export params P - array with parameters -- ALGLIB -- Copyright 20.06.2013 by Bochkanov Sergey *************************************************************************/ public static void mlpimporttunableparameters(multilayerperceptron network, double[] p, alglib.xparams _params) { int i = 0; int k = 0; int nin = 0; int nout = 0; int wcount = 0; alglib.ap.assert(alglib.ap.len(network.structinfo)>0 && alglib.ap.len(network.structinfo)>=network.structinfo[0], "MLPImportTunableParameters: Network is uninitialized"); mlpproperties(network, ref nin, ref nout, ref wcount, _params); if( mlpissoftmax(network, _params) ) { k = 0; for(i=0; i<=wcount-1; i++) { network.weights[i] = p[k]; k = k+1; } for(i=0; i<=nin-1; i++) { network.columnmeans[i] = p[k]; k = k+1; network.columnsigmas[i] = p[k]; k = k+1; } } else { k = 0; for(i=0; i<=wcount-1; i++) { network.weights[i] = p[k]; k = k+1; } for(i=0; i<=nin+nout-1; i++) { network.columnmeans[i] = p[k]; k = k+1; network.columnsigmas[i] = p[k]; k = k+1; } } } /************************************************************************* Serialization of MultiLayerPerceptron strucure INPUT PARAMETERS: Network - original OUTPUT PARAMETERS: RA - array of real numbers which stores network, array[0..RLen-1] RLen - RA lenght -- ALGLIB -- Copyright 29.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpserializeold(multilayerperceptron network, ref double[] ra, ref int rlen, alglib.xparams _params) { int i = 0; int ssize = 0; int nin = 0; int nout = 0; int wcount = 0; int sigmalen = 0; int offs = 0; int i_ = 0; int i1_ = 0; ra = new double[0]; rlen = 0; // // Unload info // ssize = network.structinfo[0]; nin = network.structinfo[1]; nout = network.structinfo[2]; wcount = network.structinfo[4]; if( mlpissoftmax(network, _params) ) { sigmalen = nin; } else { sigmalen = nin+nout; } // // RA format: // LEN DESRC. // 1 RLen // 1 version (MLPVNum) // 1 StructInfo size // SSize StructInfo // WCount Weights // SigmaLen ColumnMeans // SigmaLen ColumnSigmas // rlen = 3+ssize+wcount+2*sigmalen; ra = new double[rlen-1+1]; ra[0] = rlen; ra[1] = mlpvnum; ra[2] = ssize; offs = 3; for(i=0; i<=ssize-1; i++) { ra[offs+i] = network.structinfo[i]; } offs = offs+ssize; i1_ = (0) - (offs); for(i_=offs; i_<=offs+wcount-1;i_++) { ra[i_] = network.weights[i_+i1_]; } offs = offs+wcount; i1_ = (0) - (offs); for(i_=offs; i_<=offs+sigmalen-1;i_++) { ra[i_] = network.columnmeans[i_+i1_]; } offs = offs+sigmalen; i1_ = (0) - (offs); for(i_=offs; i_<=offs+sigmalen-1;i_++) { ra[i_] = network.columnsigmas[i_+i1_]; } offs = offs+sigmalen; } /************************************************************************* Unserialization of MultiLayerPerceptron strucure INPUT PARAMETERS: RA - real array which stores network OUTPUT PARAMETERS: Network - restored network -- ALGLIB -- Copyright 29.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpunserializeold(double[] ra, multilayerperceptron network, alglib.xparams _params) { int i = 0; int ssize = 0; int ntotal = 0; int nin = 0; int nout = 0; int wcount = 0; int sigmalen = 0; int offs = 0; int i_ = 0; int i1_ = 0; alglib.ap.assert((int)Math.Round(ra[1])==mlpvnum, "MLPUnserialize: incorrect array!"); // // Unload StructInfo from IA // offs = 3; ssize = (int)Math.Round(ra[2]); network.structinfo = new int[ssize-1+1]; for(i=0; i<=ssize-1; i++) { network.structinfo[i] = (int)Math.Round(ra[offs+i]); } offs = offs+ssize; // // Unload info from StructInfo // ssize = network.structinfo[0]; nin = network.structinfo[1]; nout = network.structinfo[2]; ntotal = network.structinfo[3]; wcount = network.structinfo[4]; if( network.structinfo[6]==0 ) { sigmalen = nin+nout; } else { sigmalen = nin; } // // Allocate space for other fields // network.weights = new double[wcount-1+1]; network.columnmeans = new double[sigmalen-1+1]; network.columnsigmas = new double[sigmalen-1+1]; network.neurons = new double[ntotal-1+1]; network.nwbuf = new double[Math.Max(wcount, 2*nout)-1+1]; network.dfdnet = new double[ntotal-1+1]; network.x = new double[nin-1+1]; network.y = new double[nout-1+1]; network.derror = new double[ntotal-1+1]; // // Copy parameters from RA // i1_ = (offs) - (0); for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = ra[i_+i1_]; } offs = offs+wcount; i1_ = (offs) - (0); for(i_=0; i_<=sigmalen-1;i_++) { network.columnmeans[i_] = ra[i_+i1_]; } offs = offs+sigmalen; i1_ = (offs) - (0); for(i_=0; i_<=sigmalen-1;i_++) { network.columnsigmas[i_] = ra[i_+i1_]; } offs = offs+sigmalen; } /************************************************************************* Randomization of neural network weights -- ALGLIB -- Copyright 06.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlprandomize(multilayerperceptron network, alglib.xparams _params) { int nin = 0; int nout = 0; int wcount = 0; int ntotal = 0; int istart = 0; hqrnd.hqrndstate r = new hqrnd.hqrndstate(); int entrysize = 0; int entryoffs = 0; int neuronidx = 0; int neurontype = 0; double vmean = 0; double vvar = 0; int i = 0; int n1 = 0; int n2 = 0; double desiredsigma = 0; int montecarlocnt = 0; double ef = 0; double ef2 = 0; double v = 0; double wscale = 0; hqrnd.hqrndrandomize(r, _params); mlpproperties(network, ref nin, ref nout, ref wcount, _params); ntotal = network.structinfo[3]; istart = network.structinfo[5]; desiredsigma = 0.5; montecarlocnt = 20; // // Stage 1: // * Network.Weights is filled by standard deviation of weights // * default values: sigma=1 // for(i=0; i<=wcount-1; i++) { network.weights[i] = 1.0; } // // Stage 2: // * assume that input neurons have zero mean and unit standard deviation // * assume that constant neurons have zero standard deviation // * perform forward pass along neurons // * for each non-input non-constant neuron: // * calculate mean and standard deviation of neuron's output // assuming that we know means/deviations of neurons which feed it // and assuming that weights has unit variance and zero mean. // * for each nonlinear neuron additionally we perform backward pass: // * scale variances of weights which feed it in such way that neuron's // input has unit standard deviation // // NOTE: this algorithm assumes that each connection feeds at most one // non-linear neuron. This assumption can be incorrect in upcoming // architectures with strong neurons. However, algorithm should // work smoothly even in this case. // // During this stage we use Network.RndBuf, which is grouped into NTotal // entries, each of them having following format: // // Buf[Offset+0] mean value of neuron's output // Buf[Offset+1] standard deviation of neuron's output // // // entrysize = 2; apserv.rvectorsetlengthatleast(ref network.rndbuf, entrysize*ntotal, _params); for(neuronidx=0; neuronidx<=ntotal-1; neuronidx++) { neurontype = network.structinfo[istart+neuronidx*nfieldwidth+0]; entryoffs = entrysize*neuronidx; if( neurontype==-2 ) { // // Input neuron: zero mean, unit variance. // network.rndbuf[entryoffs+0] = 0.0; network.rndbuf[entryoffs+1] = 1.0; continue; } if( neurontype==-3 ) { // // "-1" neuron: mean=-1, zero variance. // network.rndbuf[entryoffs+0] = -1.0; network.rndbuf[entryoffs+1] = 0.0; continue; } if( neurontype==-4 ) { // // "0" neuron: mean=0, zero variance. // network.rndbuf[entryoffs+0] = 0.0; network.rndbuf[entryoffs+1] = 0.0; continue; } if( neurontype==0 ) { // // Adaptive summator neuron: // * calculate its mean and variance. // * we assume that weights of this neuron have unit variance and zero mean. // * thus, neuron's output is always have zero mean // * as for variance, it is a bit more interesting: // * let n[i] is i-th input neuron // * let w[i] is i-th weight // * we assume that n[i] and w[i] are independently distributed // * Var(n0*w0+n1*w1+...) = Var(n0*w0)+Var(n1*w1)+... // * Var(X*Y) = mean(X)^2*Var(Y) + mean(Y)^2*Var(X) + Var(X)*Var(Y) // * mean(w[i])=0, var(w[i])=1 // * Var(n[i]*w[i]) = mean(n[i])^2 + Var(n[i]) // n1 = network.structinfo[istart+neuronidx*nfieldwidth+2]; n2 = n1+network.structinfo[istart+neuronidx*nfieldwidth+1]-1; vmean = 0.0; vvar = 0.0; for(i=n1; i<=n2; i++) { vvar = vvar+math.sqr(network.rndbuf[entrysize*i+0])+math.sqr(network.rndbuf[entrysize*i+1]); } network.rndbuf[entryoffs+0] = vmean; network.rndbuf[entryoffs+1] = Math.Sqrt(vvar); continue; } if( neurontype==-5 ) { // // Linear activation function // i = network.structinfo[istart+neuronidx*nfieldwidth+2]; vmean = network.rndbuf[entrysize*i+0]; vvar = math.sqr(network.rndbuf[entrysize*i+1]); if( (double)(vvar)>(double)(0) ) { wscale = desiredsigma/Math.Sqrt(vvar); } else { wscale = 1.0; } randomizebackwardpass(network, i, wscale, _params); network.rndbuf[entryoffs+0] = vmean*wscale; network.rndbuf[entryoffs+1] = desiredsigma; continue; } if( neurontype>0 ) { // // Nonlinear activation function: // * scale its inputs // * estimate mean/sigma of its output using Monte-Carlo method // (we simulate different inputs with unit deviation and // sample activation function output on such inputs) // i = network.structinfo[istart+neuronidx*nfieldwidth+2]; vmean = network.rndbuf[entrysize*i+0]; vvar = math.sqr(network.rndbuf[entrysize*i+1]); if( (double)(vvar)>(double)(0) ) { wscale = desiredsigma/Math.Sqrt(vvar); } else { wscale = 1.0; } randomizebackwardpass(network, i, wscale, _params); ef = 0.0; ef2 = 0.0; vmean = vmean*wscale; for(i=0; i<=montecarlocnt-1; i++) { v = vmean+desiredsigma*hqrnd.hqrndnormal(r, _params); ef = ef+v; ef2 = ef2+v*v; } ef = ef/montecarlocnt; ef2 = ef2/montecarlocnt; network.rndbuf[entryoffs+0] = ef; network.rndbuf[entryoffs+1] = Math.Max(ef2-ef*ef, 0.0); continue; } alglib.ap.assert(false, "MLPRandomize: unexpected neuron type"); } // // Stage 3: generate weights. // for(i=0; i<=wcount-1; i++) { network.weights[i] = network.weights[i]*hqrnd.hqrndnormal(r, _params); } } /************************************************************************* Randomization of neural network weights and standartisator -- ALGLIB -- Copyright 10.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlprandomizefull(multilayerperceptron network, alglib.xparams _params) { int i = 0; int nin = 0; int nout = 0; int wcount = 0; int ntotal = 0; int istart = 0; int offs = 0; int ntype = 0; mlpproperties(network, ref nin, ref nout, ref wcount, _params); ntotal = network.structinfo[3]; istart = network.structinfo[5]; // // Process network // mlprandomize(network, _params); for(i=0; i<=nin-1; i++) { network.columnmeans[i] = math.randomreal()-0.5; network.columnsigmas[i] = math.randomreal()+0.5; } if( !mlpissoftmax(network, _params) ) { for(i=0; i<=nout-1; i++) { offs = istart+(ntotal-nout+i)*nfieldwidth; ntype = network.structinfo[offs+0]; if( ntype==0 ) { // // Shifts are changed only for linear outputs neurons // network.columnmeans[nin+i] = 2*math.randomreal()-1; } if( ntype==0 || ntype==3 ) { // // Scales are changed only for linear or bounded outputs neurons. // Note that scale randomization preserves sign. // network.columnsigmas[nin+i] = Math.Sign(network.columnsigmas[nin+i])*(1.5*math.randomreal()+0.5); } } } } /************************************************************************* Internal subroutine. -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ public static void mlpinitpreprocessor(multilayerperceptron network, double[,] xy, int ssize, alglib.xparams _params) { int i = 0; int j = 0; int jmax = 0; int nin = 0; int nout = 0; int wcount = 0; int ntotal = 0; int istart = 0; int offs = 0; int ntype = 0; double[] means = new double[0]; double[] sigmas = new double[0]; double s = 0; mlpproperties(network, ref nin, ref nout, ref wcount, _params); ntotal = network.structinfo[3]; istart = network.structinfo[5]; // // Means/Sigmas // if( mlpissoftmax(network, _params) ) { jmax = nin-1; } else { jmax = nin+nout-1; } means = new double[jmax+1]; sigmas = new double[jmax+1]; for(i=0; i<=jmax; i++) { means[i] = 0; sigmas[i] = 0; } for(i=0; i<=ssize-1; i++) { for(j=0; j<=jmax; j++) { means[j] = means[j]+xy[i,j]; } } for(i=0; i<=jmax; i++) { means[i] = means[i]/ssize; } for(i=0; i<=ssize-1; i++) { for(j=0; j<=jmax; j++) { sigmas[j] = sigmas[j]+math.sqr(xy[i,j]-means[j]); } } for(i=0; i<=jmax; i++) { sigmas[i] = Math.Sqrt(sigmas[i]/ssize); } // // Inputs // for(i=0; i<=nin-1; i++) { network.columnmeans[i] = means[i]; network.columnsigmas[i] = sigmas[i]; if( (double)(network.columnsigmas[i])==(double)(0) ) { network.columnsigmas[i] = 1; } } // // Outputs // if( !mlpissoftmax(network, _params) ) { for(i=0; i<=nout-1; i++) { offs = istart+(ntotal-nout+i)*nfieldwidth; ntype = network.structinfo[offs+0]; // // Linear outputs // if( ntype==0 ) { network.columnmeans[nin+i] = means[nin+i]; network.columnsigmas[nin+i] = sigmas[nin+i]; if( (double)(network.columnsigmas[nin+i])==(double)(0) ) { network.columnsigmas[nin+i] = 1; } } // // Bounded outputs (half-interval) // if( ntype==3 ) { s = means[nin+i]-network.columnmeans[nin+i]; if( (double)(s)==(double)(0) ) { s = Math.Sign(network.columnsigmas[nin+i]); } if( (double)(s)==(double)(0) ) { s = 1.0; } network.columnsigmas[nin+i] = Math.Sign(network.columnsigmas[nin+i])*Math.Abs(s); if( (double)(network.columnsigmas[nin+i])==(double)(0) ) { network.columnsigmas[nin+i] = 1; } } } } } /************************************************************************* Internal subroutine. Initialization for preprocessor based on a sample. INPUT Network - initialized neural network; XY - sample, given by sparse matrix; SSize - sample size. OUTPUT Network - neural network with initialised preprocessor. -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpinitpreprocessorsparse(multilayerperceptron network, sparse.sparsematrix xy, int ssize, alglib.xparams _params) { int jmax = 0; int nin = 0; int nout = 0; int wcount = 0; int ntotal = 0; int istart = 0; int offs = 0; int ntype = 0; double[] means = new double[0]; double[] sigmas = new double[0]; double s = 0; int i = 0; int j = 0; mlpproperties(network, ref nin, ref nout, ref wcount, _params); ntotal = network.structinfo[3]; istart = network.structinfo[5]; // // Means/Sigmas // if( mlpissoftmax(network, _params) ) { jmax = nin-1; } else { jmax = nin+nout-1; } means = new double[jmax+1]; sigmas = new double[jmax+1]; for(i=0; i<=jmax; i++) { means[i] = 0; sigmas[i] = 0; } for(i=0; i<=ssize-1; i++) { sparse.sparsegetrow(xy, i, ref network.xyrow, _params); for(j=0; j<=jmax; j++) { means[j] = means[j]+network.xyrow[j]; } } for(i=0; i<=jmax; i++) { means[i] = means[i]/ssize; } for(i=0; i<=ssize-1; i++) { sparse.sparsegetrow(xy, i, ref network.xyrow, _params); for(j=0; j<=jmax; j++) { sigmas[j] = sigmas[j]+math.sqr(network.xyrow[j]-means[j]); } } for(i=0; i<=jmax; i++) { sigmas[i] = Math.Sqrt(sigmas[i]/ssize); } // // Inputs // for(i=0; i<=nin-1; i++) { network.columnmeans[i] = means[i]; network.columnsigmas[i] = sigmas[i]; if( (double)(network.columnsigmas[i])==(double)(0) ) { network.columnsigmas[i] = 1; } } // // Outputs // if( !mlpissoftmax(network, _params) ) { for(i=0; i<=nout-1; i++) { offs = istart+(ntotal-nout+i)*nfieldwidth; ntype = network.structinfo[offs+0]; // // Linear outputs // if( ntype==0 ) { network.columnmeans[nin+i] = means[nin+i]; network.columnsigmas[nin+i] = sigmas[nin+i]; if( (double)(network.columnsigmas[nin+i])==(double)(0) ) { network.columnsigmas[nin+i] = 1; } } // // Bounded outputs (half-interval) // if( ntype==3 ) { s = means[nin+i]-network.columnmeans[nin+i]; if( (double)(s)==(double)(0) ) { s = Math.Sign(network.columnsigmas[nin+i]); } if( (double)(s)==(double)(0) ) { s = 1.0; } network.columnsigmas[nin+i] = Math.Sign(network.columnsigmas[nin+i])*Math.Abs(s); if( (double)(network.columnsigmas[nin+i])==(double)(0) ) { network.columnsigmas[nin+i] = 1; } } } } } /************************************************************************* Internal subroutine. Initialization for preprocessor based on a subsample. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset; one sample = one row; first NIn columns contain inputs, next NOut columns - desired outputs. SetSize - real size of XY, SetSize>=0; Idx - subset of SubsetSize elements, array[SubsetSize]: * Idx[I] stores row index in the original dataset which is given by XY. Gradient is calculated with respect to rows whose indexes are stored in Idx[]. * Idx[] must store correct indexes; this function throws an exception in case incorrect index (less than 0 or larger than rows(XY)) is given * Idx[] may store indexes in any order and even with repetitions. SubsetSize- number of elements in Idx[] array. OUTPUT: Network - neural network with initialised preprocessor. NOTE: when SubsetSize<0 is used full dataset by call MLPInitPreprocessor function. -- ALGLIB -- Copyright 23.08.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpinitpreprocessorsubset(multilayerperceptron network, double[,] xy, int setsize, int[] idx, int subsetsize, alglib.xparams _params) { int jmax = 0; int nin = 0; int nout = 0; int wcount = 0; int ntotal = 0; int istart = 0; int offs = 0; int ntype = 0; double[] means = new double[0]; double[] sigmas = new double[0]; double s = 0; int npoints = 0; int i = 0; int j = 0; alglib.ap.assert(setsize>=0, "MLPInitPreprocessorSubset: SetSize<0"); if( subsetsize<0 ) { mlpinitpreprocessor(network, xy, setsize, _params); return; } alglib.ap.assert(subsetsize<=alglib.ap.len(idx), "MLPInitPreprocessorSubset: SubsetSize>Length(Idx)"); npoints = setsize; for(i=0; i<=subsetsize-1; i++) { alglib.ap.assert(idx[i]>=0, "MLPInitPreprocessorSubset: incorrect index of XY row(Idx[I]<0)"); alglib.ap.assert(idx[i]<=npoints-1, "MLPInitPreprocessorSubset: incorrect index of XY row(Idx[I]>Rows(XY)-1)"); } mlpproperties(network, ref nin, ref nout, ref wcount, _params); ntotal = network.structinfo[3]; istart = network.structinfo[5]; // // Means/Sigmas // if( mlpissoftmax(network, _params) ) { jmax = nin-1; } else { jmax = nin+nout-1; } means = new double[jmax+1]; sigmas = new double[jmax+1]; for(i=0; i<=jmax; i++) { means[i] = 0; sigmas[i] = 0; } for(i=0; i<=subsetsize-1; i++) { for(j=0; j<=jmax; j++) { means[j] = means[j]+xy[idx[i],j]; } } for(i=0; i<=jmax; i++) { means[i] = means[i]/subsetsize; } for(i=0; i<=subsetsize-1; i++) { for(j=0; j<=jmax; j++) { sigmas[j] = sigmas[j]+math.sqr(xy[idx[i],j]-means[j]); } } for(i=0; i<=jmax; i++) { sigmas[i] = Math.Sqrt(sigmas[i]/subsetsize); } // // Inputs // for(i=0; i<=nin-1; i++) { network.columnmeans[i] = means[i]; network.columnsigmas[i] = sigmas[i]; if( (double)(network.columnsigmas[i])==(double)(0) ) { network.columnsigmas[i] = 1; } } // // Outputs // if( !mlpissoftmax(network, _params) ) { for(i=0; i<=nout-1; i++) { offs = istart+(ntotal-nout+i)*nfieldwidth; ntype = network.structinfo[offs+0]; // // Linear outputs // if( ntype==0 ) { network.columnmeans[nin+i] = means[nin+i]; network.columnsigmas[nin+i] = sigmas[nin+i]; if( (double)(network.columnsigmas[nin+i])==(double)(0) ) { network.columnsigmas[nin+i] = 1; } } // // Bounded outputs (half-interval) // if( ntype==3 ) { s = means[nin+i]-network.columnmeans[nin+i]; if( (double)(s)==(double)(0) ) { s = Math.Sign(network.columnsigmas[nin+i]); } if( (double)(s)==(double)(0) ) { s = 1.0; } network.columnsigmas[nin+i] = Math.Sign(network.columnsigmas[nin+i])*Math.Abs(s); if( (double)(network.columnsigmas[nin+i])==(double)(0) ) { network.columnsigmas[nin+i] = 1; } } } } } /************************************************************************* Internal subroutine. Initialization for preprocessor based on a subsample. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset, given by sparse matrix; one sample = one row; first NIn columns contain inputs, next NOut columns - desired outputs. SetSize - real size of XY, SetSize>=0; Idx - subset of SubsetSize elements, array[SubsetSize]: * Idx[I] stores row index in the original dataset which is given by XY. Gradient is calculated with respect to rows whose indexes are stored in Idx[]. * Idx[] must store correct indexes; this function throws an exception in case incorrect index (less than 0 or larger than rows(XY)) is given * Idx[] may store indexes in any order and even with repetitions. SubsetSize- number of elements in Idx[] array. OUTPUT: Network - neural network with initialised preprocessor. NOTE: when SubsetSize<0 is used full dataset by call MLPInitPreprocessorSparse function. -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpinitpreprocessorsparsesubset(multilayerperceptron network, sparse.sparsematrix xy, int setsize, int[] idx, int subsetsize, alglib.xparams _params) { int jmax = 0; int nin = 0; int nout = 0; int wcount = 0; int ntotal = 0; int istart = 0; int offs = 0; int ntype = 0; double[] means = new double[0]; double[] sigmas = new double[0]; double s = 0; int npoints = 0; int i = 0; int j = 0; alglib.ap.assert(setsize>=0, "MLPInitPreprocessorSparseSubset: SetSize<0"); if( subsetsize<0 ) { mlpinitpreprocessorsparse(network, xy, setsize, _params); return; } alglib.ap.assert(subsetsize<=alglib.ap.len(idx), "MLPInitPreprocessorSparseSubset: SubsetSize>Length(Idx)"); npoints = setsize; for(i=0; i<=subsetsize-1; i++) { alglib.ap.assert(idx[i]>=0, "MLPInitPreprocessorSparseSubset: incorrect index of XY row(Idx[I]<0)"); alglib.ap.assert(idx[i]<=npoints-1, "MLPInitPreprocessorSparseSubset: incorrect index of XY row(Idx[I]>Rows(XY)-1)"); } mlpproperties(network, ref nin, ref nout, ref wcount, _params); ntotal = network.structinfo[3]; istart = network.structinfo[5]; // // Means/Sigmas // if( mlpissoftmax(network, _params) ) { jmax = nin-1; } else { jmax = nin+nout-1; } means = new double[jmax+1]; sigmas = new double[jmax+1]; for(i=0; i<=jmax; i++) { means[i] = 0; sigmas[i] = 0; } for(i=0; i<=subsetsize-1; i++) { sparse.sparsegetrow(xy, idx[i], ref network.xyrow, _params); for(j=0; j<=jmax; j++) { means[j] = means[j]+network.xyrow[j]; } } for(i=0; i<=jmax; i++) { means[i] = means[i]/subsetsize; } for(i=0; i<=subsetsize-1; i++) { sparse.sparsegetrow(xy, idx[i], ref network.xyrow, _params); for(j=0; j<=jmax; j++) { sigmas[j] = sigmas[j]+math.sqr(network.xyrow[j]-means[j]); } } for(i=0; i<=jmax; i++) { sigmas[i] = Math.Sqrt(sigmas[i]/subsetsize); } // // Inputs // for(i=0; i<=nin-1; i++) { network.columnmeans[i] = means[i]; network.columnsigmas[i] = sigmas[i]; if( (double)(network.columnsigmas[i])==(double)(0) ) { network.columnsigmas[i] = 1; } } // // Outputs // if( !mlpissoftmax(network, _params) ) { for(i=0; i<=nout-1; i++) { offs = istart+(ntotal-nout+i)*nfieldwidth; ntype = network.structinfo[offs+0]; // // Linear outputs // if( ntype==0 ) { network.columnmeans[nin+i] = means[nin+i]; network.columnsigmas[nin+i] = sigmas[nin+i]; if( (double)(network.columnsigmas[nin+i])==(double)(0) ) { network.columnsigmas[nin+i] = 1; } } // // Bounded outputs (half-interval) // if( ntype==3 ) { s = means[nin+i]-network.columnmeans[nin+i]; if( (double)(s)==(double)(0) ) { s = Math.Sign(network.columnsigmas[nin+i]); } if( (double)(s)==(double)(0) ) { s = 1.0; } network.columnsigmas[nin+i] = Math.Sign(network.columnsigmas[nin+i])*Math.Abs(s); if( (double)(network.columnsigmas[nin+i])==(double)(0) ) { network.columnsigmas[nin+i] = 1; } } } } } /************************************************************************* Returns information about initialized network: number of inputs, outputs, weights. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpproperties(multilayerperceptron network, ref int nin, ref int nout, ref int wcount, alglib.xparams _params) { nin = 0; nout = 0; wcount = 0; nin = network.structinfo[1]; nout = network.structinfo[2]; wcount = network.structinfo[4]; } /************************************************************************* Returns number of "internal", low-level neurons in the network (one which is stored in StructInfo). -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static int mlpntotal(multilayerperceptron network, alglib.xparams _params) { int result = 0; result = network.structinfo[3]; return result; } /************************************************************************* Returns number of inputs. -- ALGLIB -- Copyright 19.10.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetinputscount(multilayerperceptron network, alglib.xparams _params) { int result = 0; result = network.structinfo[1]; return result; } /************************************************************************* Returns number of outputs. -- ALGLIB -- Copyright 19.10.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetoutputscount(multilayerperceptron network, alglib.xparams _params) { int result = 0; result = network.structinfo[2]; return result; } /************************************************************************* Returns number of weights. -- ALGLIB -- Copyright 19.10.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetweightscount(multilayerperceptron network, alglib.xparams _params) { int result = 0; result = network.structinfo[4]; return result; } /************************************************************************* Tells whether network is SOFTMAX-normalized (i.e. classifier) or not. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static bool mlpissoftmax(multilayerperceptron network, alglib.xparams _params) { bool result = new bool(); result = network.structinfo[6]==1; return result; } /************************************************************************* This function returns total number of layers (including input, hidden and output layers). -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetlayerscount(multilayerperceptron network, alglib.xparams _params) { int result = 0; result = alglib.ap.len(network.hllayersizes); return result; } /************************************************************************* This function returns size of K-th layer. K=0 corresponds to input layer, K=CNT-1 corresponds to output layer. Size of the output layer is always equal to the number of outputs, although when we have softmax-normalized network, last neuron doesn't have any connections - it is just zero. -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static int mlpgetlayersize(multilayerperceptron network, int k, alglib.xparams _params) { int result = 0; alglib.ap.assert(k>=0 && k=0 && i=0 && i=0, "MLPGetNeuronInfo: incorrect (nonexistent) layer or neuron index"); // // 1. find offset of the activation function record in the // if( network.hlneurons[highlevelidx*hlnfieldwidth+2]>=0 ) { activationoffset = istart+network.hlneurons[highlevelidx*hlnfieldwidth+2]*nfieldwidth; fkind = network.structinfo[activationoffset+0]; } else { fkind = 0; } if( network.hlneurons[highlevelidx*hlnfieldwidth+3]>=0 ) { threshold = network.weights[network.hlneurons[highlevelidx*hlnfieldwidth+3]]; } else { threshold = 0; } } /************************************************************************* This function returns information about connection from I0-th neuron of K0-th layer to I1-th neuron of K1-th layer. INPUT PARAMETERS: Network - network K0 - layer index I0 - neuron index (within layer) K1 - layer index I1 - neuron index (within layer) RESULT: connection weight (zero for non-existent connections) This function: 1. throws exception if layer or neuron with given index do not exists. 2. returns zero if neurons exist, but there is no connection between them -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static double mlpgetweight(multilayerperceptron network, int k0, int i0, int k1, int i1, alglib.xparams _params) { double result = 0; int ccnt = 0; int highlevelidx = 0; ccnt = alglib.ap.len(network.hlconnections)/hlconnfieldwidth; // // check params // alglib.ap.assert(k0>=0 && k0=0 && i0=0 && k1=0 && i1=0 ) { result = network.weights[network.hlconnections[highlevelidx*hlconnfieldwidth+4]]; } else { result = 0; } return result; } /************************************************************************* This function sets offset/scaling coefficients for I-th input of the network. INPUT PARAMETERS: Network - network I - input index Mean - mean term Sigma - sigma term (if zero, will be replaced by 1.0) NTE: I-th input is passed through linear transformation IN[i] = (IN[i]-Mean)/Sigma before feeding to the network. This function sets Mean and Sigma. -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpsetinputscaling(multilayerperceptron network, int i, double mean, double sigma, alglib.xparams _params) { alglib.ap.assert(i>=0 && i=0 && i=0, "MLPSetNeuronInfo: incorrect (nonexistent) layer or neuron index"); // // activation function // if( network.hlneurons[highlevelidx*hlnfieldwidth+2]>=0 ) { activationoffset = istart+network.hlneurons[highlevelidx*hlnfieldwidth+2]*nfieldwidth; network.structinfo[activationoffset+0] = fkind; } else { alglib.ap.assert(fkind==0, "MLPSetNeuronInfo: you try to set activation function for neuron which can not have one"); } // // Threshold // if( network.hlneurons[highlevelidx*hlnfieldwidth+3]>=0 ) { network.weights[network.hlneurons[highlevelidx*hlnfieldwidth+3]] = threshold; } else { alglib.ap.assert((double)(threshold)==(double)(0), "MLPSetNeuronInfo: you try to set non-zero threshold for neuron which can not have one"); } } /************************************************************************* This function modifies information about connection from I0-th neuron of K0-th layer to I1-th neuron of K1-th layer. INPUT PARAMETERS: Network - network K0 - layer index I0 - neuron index (within layer) K1 - layer index I1 - neuron index (within layer) W - connection weight (must be zero for non-existent connections) This function: 1. throws exception if layer or neuron with given index do not exists. 2. throws exception if you try to set non-zero weight for non-existent connection -- ALGLIB -- Copyright 25.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpsetweight(multilayerperceptron network, int k0, int i0, int k1, int i1, double w, alglib.xparams _params) { int ccnt = 0; int highlevelidx = 0; ccnt = alglib.ap.len(network.hlconnections)/hlconnfieldwidth; // // check params // alglib.ap.assert(k0>=0 && k0=0 && i0=0 && k1=0 && i1=0 ) { network.weights[network.hlconnections[highlevelidx*hlconnfieldwidth+4]] = w; } else { alglib.ap.assert((double)(w)==(double)(0), "MLPSetWeight: you try to set non-zero weight for non-existent connection"); } } /************************************************************************* Neural network activation function INPUT PARAMETERS: NET - neuron input K - function index (zero for linear function) OUTPUT PARAMETERS: F - function DF - its derivative D2F - its second derivative -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpactivationfunction(double net, int k, ref double f, ref double df, ref double d2f, alglib.xparams _params) { double net2 = 0; double arg = 0; double root = 0; double r = 0; f = 0; df = 0; d2f = 0; if( k==0 || k==-5 ) { f = net; df = 1; d2f = 0; return; } if( k==1 ) { // // TanH activation function // if( (double)(Math.Abs(net))<(double)(100) ) { f = Math.Tanh(net); } else { f = Math.Sign(net); } df = 1-f*f; d2f = -(2*f*df); return; } if( k==3 ) { // // EX activation function // if( (double)(net)>=(double)(0) ) { net2 = net*net; arg = net2+1; root = Math.Sqrt(arg); f = net+root; r = net/root; df = 1+r; d2f = (root-net*r)/arg; } else { f = Math.Exp(net); df = f; d2f = f; } return; } if( k==2 ) { f = Math.Exp(-math.sqr(net)); df = -(2*net*f); d2f = -(2*(f+df*net)); return; } f = 0; df = 0; d2f = 0; } /************************************************************************* Procesing INPUT PARAMETERS: Network - neural network X - input vector, array[0..NIn-1]. OUTPUT PARAMETERS: Y - result. Regression estimate when solving regression task, vector of posterior probabilities for classification task. See also MLPProcessI -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpprocess(multilayerperceptron network, double[] x, ref double[] y, alglib.xparams _params) { if( alglib.ap.len(y)=npoints, "MLPError: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+1, "MLPError: XY has less than NIn+1 columns"); } else { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPError: XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, xy, network.dummysxy, npoints, 0, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = math.sqr(network.err.rmserror)*npoints*mlpgetoutputscount(network, _params)/2; return result; } /************************************************************************* Error of the neural network on dataset given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0 RESULT: sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static double mlperrorsparse(multilayerperceptron network, sparse.sparsematrix xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPErrorSparse: XY is not in CRS format."); alglib.ap.assert(sparse.sparsegetnrows(xy, _params)>=npoints, "MLPErrorSparse: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+1, "MLPErrorSparse: XY has less than NIn+1 columns"); } else { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPErrorSparse: XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, network.dummydxy, xy, npoints, 1, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = math.sqr(network.err.rmserror)*npoints*mlpgetoutputscount(network, _params)/2; return result; } /************************************************************************* Natural error function for neural network, internal subroutine. NOTE: this function is single-threaded. Unlike other error function, it receives no speed-up from being executed in SMP mode. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static double mlperrorn(multilayerperceptron network, double[,] xy, int ssize, alglib.xparams _params) { double result = 0; int i = 0; int k = 0; int nin = 0; int nout = 0; int wcount = 0; double e = 0; int i_ = 0; int i1_ = 0; mlpproperties(network, ref nin, ref nout, ref wcount, _params); result = 0; for(i=0; i<=ssize-1; i++) { // // Process vector // for(i_=0; i_<=nin-1;i_++) { network.x[i_] = xy[i,i_]; } mlpprocess(network, network.x, ref network.y, _params); // // Update error function // if( network.structinfo[6]==0 ) { // // Least squares error function // i1_ = (nin) - (0); for(i_=0; i_<=nout-1;i_++) { network.y[i_] = network.y[i_] - xy[i,i_+i1_]; } e = 0.0; for(i_=0; i_<=nout-1;i_++) { e += network.y[i_]*network.y[i_]; } result = result+e/2; } else { // // Cross-entropy error function // k = (int)Math.Round(xy[i,nin]); if( k>=0 && k=npoints, "MLPClsError: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+1, "MLPClsError: XY has less than NIn+1 columns"); } else { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPClsError: XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, xy, network.dummysxy, npoints, 0, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = (int)Math.Round(npoints*network.err.relclserror); return result; } /************************************************************************* Relative classification error on the test set. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: Percent of incorrectly classified cases. Works both for classifier networks and general purpose networks used as classifiers. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 25.12.2008 by Bochkanov Sergey *************************************************************************/ public static double mlprelclserror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "MLPRelClsError: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+1, "MLPRelClsError: XY has less than NIn+1 columns"); } else { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPRelClsError: XY has less than NIn+NOut columns"); } } if( npoints>0 ) { result = (double)mlpclserror(network, xy, npoints, _params)/(double)npoints; } else { result = 0.0; } return result; } /************************************************************************* Relative classification error on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: Percent of incorrectly classified cases. Works both for classifier networks and general purpose networks used as classifiers. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 09.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlprelclserrorsparse(multilayerperceptron network, sparse.sparsematrix xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPRelClsErrorSparse: sparse matrix XY is not in CRS format."); alglib.ap.assert(sparse.sparsegetnrows(xy, _params)>=npoints, "MLPRelClsErrorSparse: sparse matrix XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+1, "MLPRelClsErrorSparse: sparse matrix XY has less than NIn+1 columns"); } else { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPRelClsErrorSparse: sparse matrix XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, network.dummydxy, xy, npoints, 1, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = network.err.relclserror; return result; } /************************************************************************* Average cross-entropy (in bits per element) on the test set. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: CrossEntropy/(NPoints*LN(2)). Zero if network solves regression task. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 08.01.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpavgce(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "MLPAvgCE: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+1, "MLPAvgCE: XY has less than NIn+1 columns"); } else { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPAvgCE: XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, xy, network.dummysxy, npoints, 0, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = network.err.avgce; return result; } /************************************************************************* Average cross-entropy (in bits per element) on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: CrossEntropy/(NPoints*LN(2)). Zero if network solves regression task. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 9.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlpavgcesparse(multilayerperceptron network, sparse.sparsematrix xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPAvgCESparse: sparse matrix XY is not in CRS format."); alglib.ap.assert(sparse.sparsegetnrows(xy, _params)>=npoints, "MLPAvgCESparse: sparse matrix XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+1, "MLPAvgCESparse: sparse matrix XY has less than NIn+1 columns"); } else { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPAvgCESparse: sparse matrix XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, network.dummydxy, xy, npoints, 1, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = network.err.avgce; return result; } /************************************************************************* RMS error on the test set given. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: Root mean square error. Its meaning for regression task is obvious. As for classification task, RMS error means error when estimating posterior probabilities. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static double mlprmserror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "MLPRMSError: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+1, "MLPRMSError: XY has less than NIn+1 columns"); } else { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPRMSError: XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, xy, network.dummysxy, npoints, 0, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = network.err.rmserror; return result; } /************************************************************************* RMS error on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: Root mean square error. Its meaning for regression task is obvious. As for classification task, RMS error means error when estimating posterior probabilities. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 09.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlprmserrorsparse(multilayerperceptron network, sparse.sparsematrix xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPRMSErrorSparse: sparse matrix XY is not in CRS format."); alglib.ap.assert(sparse.sparsegetnrows(xy, _params)>=npoints, "MLPRMSErrorSparse: sparse matrix XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+1, "MLPRMSErrorSparse: sparse matrix XY has less than NIn+1 columns"); } else { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPRMSErrorSparse: sparse matrix XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, network.dummydxy, xy, npoints, 1, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = network.err.rmserror; return result; } /************************************************************************* Average absolute error on the test set. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: Its meaning for regression task is obvious. As for classification task, it means average error when estimating posterior probabilities. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 11.03.2008 by Bochkanov Sergey *************************************************************************/ public static double mlpavgerror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "MLPAvgError: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+1, "MLPAvgError: XY has less than NIn+1 columns"); } else { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPAvgError: XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, xy, network.dummysxy, npoints, 0, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = network.err.avgerror; return result; } /************************************************************************* Average absolute error on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: Its meaning for regression task is obvious. As for classification task, it means average error when estimating posterior probabilities. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 09.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlpavgerrorsparse(multilayerperceptron network, sparse.sparsematrix xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPAvgErrorSparse: XY is not in CRS format."); alglib.ap.assert(sparse.sparsegetnrows(xy, _params)>=npoints, "MLPAvgErrorSparse: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+1, "MLPAvgErrorSparse: XY has less than NIn+1 columns"); } else { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPAvgErrorSparse: XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, network.dummydxy, xy, npoints, 1, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = network.err.avgerror; return result; } /************************************************************************* Average relative error on the test set. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; NPoints - points count. RESULT: Its meaning for regression task is obvious. As for classification task, it means average relative error when estimating posterior probability of belonging to the correct class. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 11.03.2008 by Bochkanov Sergey *************************************************************************/ public static double mlpavgrelerror(multilayerperceptron network, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "MLPAvgRelError: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+1, "MLPAvgRelError: XY has less than NIn+1 columns"); } else { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPAvgRelError: XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, xy, network.dummysxy, npoints, 0, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = network.err.avgrelerror; return result; } /************************************************************************* Average relative error on the test set given by sparse matrix. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. NPoints - points count, >=0. RESULT: Its meaning for regression task is obvious. As for classification task, it means average relative error when estimating posterior probability of belonging to the correct class. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 09.08.2012 by Bochkanov Sergey *************************************************************************/ public static double mlpavgrelerrorsparse(multilayerperceptron network, sparse.sparsematrix xy, int npoints, alglib.xparams _params) { double result = 0; alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPAvgRelErrorSparse: XY is not in CRS format."); alglib.ap.assert(sparse.sparsegetnrows(xy, _params)>=npoints, "MLPAvgRelErrorSparse: XY has less than NPoints rows"); if( npoints>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+1, "MLPAvgRelErrorSparse: XY has less than NIn+1 columns"); } else { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPAvgRelErrorSparse: XY has less than NIn+NOut columns"); } } mlpallerrorsx(network, network.dummydxy, xy, npoints, 1, network.dummyidx, 0, npoints, 0, network.buf, network.err, _params); result = network.err.avgrelerror; return result; } /************************************************************************* Gradient calculation INPUT PARAMETERS: Network - network initialized with one of the network creation funcs X - input vector, length of array must be at least NIn DesiredY- desired outputs, length of array must be at least NOut Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpgrad(multilayerperceptron network, double[] x, double[] desiredy, ref double e, ref double[] grad, alglib.xparams _params) { int i = 0; int nout = 0; int ntotal = 0; e = 0; // // Alloc // apserv.rvectorsetlengthatleast(ref grad, network.structinfo[4], _params); // // Prepare dError/dOut, internal structures // mlpprocess(network, x, ref network.y, _params); nout = network.structinfo[2]; ntotal = network.structinfo[3]; e = 0; for(i=0; i<=ntotal-1; i++) { network.derror[i] = 0; } for(i=0; i<=nout-1; i++) { network.derror[ntotal-nout+i] = network.y[i]-desiredy[i]; e = e+math.sqr(network.y[i]-desiredy[i])/2; } // // gradient // mlpinternalcalculategradient(network, network.neurons, network.weights, ref network.derror, ref grad, false, _params); } /************************************************************************* Gradient calculation (natural error function is used) INPUT PARAMETERS: Network - network initialized with one of the network creation funcs X - input vector, length of array must be at least NIn DesiredY- desired outputs, length of array must be at least NOut Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, sum-of-squares for regression networks, cross-entropy for classification networks. Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpgradn(multilayerperceptron network, double[] x, double[] desiredy, ref double e, ref double[] grad, alglib.xparams _params) { double s = 0; int i = 0; int nout = 0; int ntotal = 0; e = 0; // // Alloc // apserv.rvectorsetlengthatleast(ref grad, network.structinfo[4], _params); // // Prepare dError/dOut, internal structures // mlpprocess(network, x, ref network.y, _params); nout = network.structinfo[2]; ntotal = network.structinfo[3]; for(i=0; i<=ntotal-1; i++) { network.derror[i] = 0; } e = 0; if( network.structinfo[6]==0 ) { // // Regression network, least squares // for(i=0; i<=nout-1; i++) { network.derror[ntotal-nout+i] = network.y[i]-desiredy[i]; e = e+math.sqr(network.y[i]-desiredy[i])/2; } } else { // // Classification network, cross-entropy // s = 0; for(i=0; i<=nout-1; i++) { s = s+desiredy[i]; } for(i=0; i<=nout-1; i++) { network.derror[ntotal-nout+i] = s*network.y[i]-desiredy[i]; e = e+safecrossentropy(desiredy[i], network.y[i], _params); } } // // gradient // mlpinternalcalculategradient(network, network.neurons, network.weights, ref network.derror, ref grad, true, _params); } /************************************************************************* Batch gradient calculation for a set of inputs/outputs ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset in dense format; one sample = one row: * first NIn columns contain inputs, * for regression problem, next NOut columns store desired outputs. * for classification problem, next column (just one!) stores class number. SSize - number of elements in XY Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpgradbatch(multilayerperceptron network, double[,] xy, int ssize, ref double e, ref double[] grad, alglib.xparams _params) { int i = 0; int nin = 0; int nout = 0; int wcount = 0; int subset0 = 0; int subset1 = 0; int subsettype = 0; smlpgrad sgrad = null; e = 0; alglib.ap.assert(ssize>=0, "MLPGradBatchSparse: SSize<0"); subset0 = 0; subset1 = ssize; subsettype = 0; mlpproperties(network, ref nin, ref nout, ref wcount, _params); apserv.rvectorsetlengthatleast(ref grad, wcount, _params); alglib.smp.ae_shared_pool_first_recycled(network.gradbuf, ref sgrad); while( sgrad!=null ) { sgrad.f = 0.0; for(i=0; i<=wcount-1; i++) { sgrad.g[i] = 0.0; } alglib.smp.ae_shared_pool_next_recycled(network.gradbuf, ref sgrad); } mlpgradbatchx(network, xy, network.dummysxy, ssize, 0, network.dummyidx, subset0, subset1, subsettype, network.buf, network.gradbuf, _params); e = 0.0; for(i=0; i<=wcount-1; i++) { grad[i] = 0.0; } alglib.smp.ae_shared_pool_first_recycled(network.gradbuf, ref sgrad); while( sgrad!=null ) { e = e+sgrad.f; for(i=0; i<=wcount-1; i++) { grad[i] = grad[i]+sgrad.g[i]; } alglib.smp.ae_shared_pool_next_recycled(network.gradbuf, ref sgrad); } } /************************************************************************* Batch gradient calculation for a set of inputs/outputs given by sparse matrices ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset in sparse format; one sample = one row: * MATRIX MUST BE STORED IN CRS FORMAT * first NIn columns contain inputs. * for regression problem, next NOut columns store desired outputs. * for classification problem, next column (just one!) stores class number. SSize - number of elements in XY Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpgradbatchsparse(multilayerperceptron network, sparse.sparsematrix xy, int ssize, ref double e, ref double[] grad, alglib.xparams _params) { int i = 0; int nin = 0; int nout = 0; int wcount = 0; int subset0 = 0; int subset1 = 0; int subsettype = 0; smlpgrad sgrad = null; e = 0; alglib.ap.assert(ssize>=0, "MLPGradBatchSparse: SSize<0"); alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPGradBatchSparse: sparse matrix XY must be in CRS format."); subset0 = 0; subset1 = ssize; subsettype = 0; mlpproperties(network, ref nin, ref nout, ref wcount, _params); apserv.rvectorsetlengthatleast(ref grad, wcount, _params); alglib.smp.ae_shared_pool_first_recycled(network.gradbuf, ref sgrad); while( sgrad!=null ) { sgrad.f = 0.0; for(i=0; i<=wcount-1; i++) { sgrad.g[i] = 0.0; } alglib.smp.ae_shared_pool_next_recycled(network.gradbuf, ref sgrad); } mlpgradbatchx(network, network.dummydxy, xy, ssize, 1, network.dummyidx, subset0, subset1, subsettype, network.buf, network.gradbuf, _params); e = 0.0; for(i=0; i<=wcount-1; i++) { grad[i] = 0.0; } alglib.smp.ae_shared_pool_first_recycled(network.gradbuf, ref sgrad); while( sgrad!=null ) { e = e+sgrad.f; for(i=0; i<=wcount-1; i++) { grad[i] = grad[i]+sgrad.g[i]; } alglib.smp.ae_shared_pool_next_recycled(network.gradbuf, ref sgrad); } } /************************************************************************* Batch gradient calculation for a subset of dataset ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset in dense format; one sample = one row: * first NIn columns contain inputs, * for regression problem, next NOut columns store desired outputs. * for classification problem, next column (just one!) stores class number. SetSize - real size of XY, SetSize>=0; Idx - subset of SubsetSize elements, array[SubsetSize]: * Idx[I] stores row index in the original dataset which is given by XY. Gradient is calculated with respect to rows whose indexes are stored in Idx[]. * Idx[] must store correct indexes; this function throws an exception in case incorrect index (less than 0 or larger than rows(XY)) is given * Idx[] may store indexes in any order and even with repetitions. SubsetSize- number of elements in Idx[] array: * positive value means that subset given by Idx[] is processed * zero value results in zero gradient * negative value means that full dataset is processed Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpgradbatchsubset(multilayerperceptron network, double[,] xy, int setsize, int[] idx, int subsetsize, ref double e, ref double[] grad, alglib.xparams _params) { int i = 0; int nin = 0; int nout = 0; int wcount = 0; int npoints = 0; int subset0 = 0; int subset1 = 0; int subsettype = 0; smlpgrad sgrad = null; e = 0; alglib.ap.assert(setsize>=0, "MLPGradBatchSubset: SetSize<0"); alglib.ap.assert(subsetsize<=alglib.ap.len(idx), "MLPGradBatchSubset: SubsetSize>Length(Idx)"); npoints = setsize; if( subsetsize<0 ) { subset0 = 0; subset1 = setsize; subsettype = 0; } else { subset0 = 0; subset1 = subsetsize; subsettype = 1; for(i=0; i<=subsetsize-1; i++) { alglib.ap.assert(idx[i]>=0, "MLPGradBatchSubset: incorrect index of XY row(Idx[I]<0)"); alglib.ap.assert(idx[i]<=npoints-1, "MLPGradBatchSubset: incorrect index of XY row(Idx[I]>Rows(XY)-1)"); } } mlpproperties(network, ref nin, ref nout, ref wcount, _params); apserv.rvectorsetlengthatleast(ref grad, wcount, _params); alglib.smp.ae_shared_pool_first_recycled(network.gradbuf, ref sgrad); while( sgrad!=null ) { sgrad.f = 0.0; for(i=0; i<=wcount-1; i++) { sgrad.g[i] = 0.0; } alglib.smp.ae_shared_pool_next_recycled(network.gradbuf, ref sgrad); } mlpgradbatchx(network, xy, network.dummysxy, setsize, 0, idx, subset0, subset1, subsettype, network.buf, network.gradbuf, _params); e = 0.0; for(i=0; i<=wcount-1; i++) { grad[i] = 0.0; } alglib.smp.ae_shared_pool_first_recycled(network.gradbuf, ref sgrad); while( sgrad!=null ) { e = e+sgrad.f; for(i=0; i<=wcount-1; i++) { grad[i] = grad[i]+sgrad.g[i]; } alglib.smp.ae_shared_pool_next_recycled(network.gradbuf, ref sgrad); } } /************************************************************************* Batch gradient calculation for a set of inputs/outputs for a subset of dataset given by set of indexes. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset in sparse format; one sample = one row: * MATRIX MUST BE STORED IN CRS FORMAT * first NIn columns contain inputs, * for regression problem, next NOut columns store desired outputs. * for classification problem, next column (just one!) stores class number. SetSize - real size of XY, SetSize>=0; Idx - subset of SubsetSize elements, array[SubsetSize]: * Idx[I] stores row index in the original dataset which is given by XY. Gradient is calculated with respect to rows whose indexes are stored in Idx[]. * Idx[] must store correct indexes; this function throws an exception in case incorrect index (less than 0 or larger than rows(XY)) is given * Idx[] may store indexes in any order and even with repetitions. SubsetSize- number of elements in Idx[] array: * positive value means that subset given by Idx[] is processed * zero value results in zero gradient * negative value means that full dataset is processed Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, SUM(sqr(y[i]-desiredy[i])/2,i) Grad - gradient of E with respect to weights of network, array[WCount] NOTE: when SubsetSize<0 is used full dataset by call MLPGradBatchSparse function. -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpgradbatchsparsesubset(multilayerperceptron network, sparse.sparsematrix xy, int setsize, int[] idx, int subsetsize, ref double e, ref double[] grad, alglib.xparams _params) { int i = 0; int nin = 0; int nout = 0; int wcount = 0; int npoints = 0; int subset0 = 0; int subset1 = 0; int subsettype = 0; smlpgrad sgrad = null; e = 0; alglib.ap.assert(setsize>=0, "MLPGradBatchSparseSubset: SetSize<0"); alglib.ap.assert(subsetsize<=alglib.ap.len(idx), "MLPGradBatchSparseSubset: SubsetSize>Length(Idx)"); alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPGradBatchSparseSubset: sparse matrix XY must be in CRS format."); npoints = setsize; if( subsetsize<0 ) { subset0 = 0; subset1 = setsize; subsettype = 0; } else { subset0 = 0; subset1 = subsetsize; subsettype = 1; for(i=0; i<=subsetsize-1; i++) { alglib.ap.assert(idx[i]>=0, "MLPGradBatchSparseSubset: incorrect index of XY row(Idx[I]<0)"); alglib.ap.assert(idx[i]<=npoints-1, "MLPGradBatchSparseSubset: incorrect index of XY row(Idx[I]>Rows(XY)-1)"); } } mlpproperties(network, ref nin, ref nout, ref wcount, _params); apserv.rvectorsetlengthatleast(ref grad, wcount, _params); alglib.smp.ae_shared_pool_first_recycled(network.gradbuf, ref sgrad); while( sgrad!=null ) { sgrad.f = 0.0; for(i=0; i<=wcount-1; i++) { sgrad.g[i] = 0.0; } alglib.smp.ae_shared_pool_next_recycled(network.gradbuf, ref sgrad); } mlpgradbatchx(network, network.dummydxy, xy, setsize, 1, idx, subset0, subset1, subsettype, network.buf, network.gradbuf, _params); e = 0.0; for(i=0; i<=wcount-1; i++) { grad[i] = 0.0; } alglib.smp.ae_shared_pool_first_recycled(network.gradbuf, ref sgrad); while( sgrad!=null ) { e = e+sgrad.f; for(i=0; i<=wcount-1; i++) { grad[i] = grad[i]+sgrad.g[i]; } alglib.smp.ae_shared_pool_next_recycled(network.gradbuf, ref sgrad); } } /************************************************************************* Internal function which actually calculates batch gradient for a subset or full dataset, which can be represented in different formats. THIS FUNCTION IS NOT INTENDED TO BE USED BY ALGLIB USERS! -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpgradbatchx(multilayerperceptron network, double[,] densexy, sparse.sparsematrix sparsexy, int datasetsize, int datasettype, int[] idx, int subset0, int subset1, int subsettype, alglib.smp.shared_pool buf, alglib.smp.shared_pool gradbuf, alglib.xparams _params) { int nin = 0; int nout = 0; int wcount = 0; int rowsize = 0; int srcidx = 0; int cstart = 0; int csize = 0; int j = 0; double problemcost = 0; hpccores.mlpbuffers buf2 = null; int len0 = 0; int len1 = 0; hpccores.mlpbuffers pbuf = null; smlpgrad sgrad = null; int i_ = 0; alglib.ap.assert(datasetsize>=0, "MLPGradBatchX: SetSize<0"); alglib.ap.assert(datasettype==0 || datasettype==1, "MLPGradBatchX: DatasetType is incorrect"); alglib.ap.assert(subsettype==0 || subsettype==1, "MLPGradBatchX: SubsetType is incorrect"); // // Determine network and dataset properties // mlpproperties(network, ref nin, ref nout, ref wcount, _params); if( mlpissoftmax(network, _params) ) { rowsize = nin+1; } else { rowsize = nin+nout; } // // Split problem. // // Splitting problem allows us to reduce effect of single-precision // arithmetics (SSE-optimized version of MLPChunkedGradient uses single // precision internally, but converts them to double precision after // results are exported from HPC buffer to network). Small batches are // calculated in single precision, results are aggregated in double // precision, and it allows us to avoid accumulation of errors when // we process very large batches (tens of thousands of items). // // NOTE: it is important to use real arithmetics for ProblemCost // because ProblemCost may be larger than MAXINT. // problemcost = subset1-subset0; problemcost = problemcost*wcount*2; if( (double)(problemcost)>=(double)(apserv.smpactivationlevel(_params)) && subset1-subset0>=2*microbatchsize ) { if( _trypexec_mlpgradbatchx(network,densexy,sparsexy,datasetsize,datasettype,idx,subset0,subset1,subsettype,buf,gradbuf, _params) ) { return; } } if( subset1-subset0>=2*microbatchsize && (double)(problemcost)>(double)(apserv.spawnlevel(_params)) ) { apserv.splitlength(subset1-subset0, microbatchsize, ref len0, ref len1, _params); mlpgradbatchx(network, densexy, sparsexy, datasetsize, datasettype, idx, subset0, subset0+len0, subsettype, buf, gradbuf, _params); mlpgradbatchx(network, densexy, sparsexy, datasetsize, datasettype, idx, subset0+len0, subset1, subsettype, buf, gradbuf, _params); return; } // // Chunked processing // alglib.smp.ae_shared_pool_retrieve(gradbuf, ref sgrad); alglib.smp.ae_shared_pool_retrieve(buf, ref pbuf); hpccores.hpcpreparechunkedgradient(network.weights, wcount, mlpntotal(network, _params), nin, nout, pbuf, _params); cstart = subset0; while( cstart=0, "MLPGradBatchX: internal error"); if( datasettype==0 ) { for(i_=0; i_<=rowsize-1;i_++) { pbuf.xy[j,i_] = densexy[srcidx,i_]; } } if( datasettype==1 ) { sparse.sparsegetrow(sparsexy, srcidx, ref pbuf.xyrow, _params); for(i_=0; i_<=rowsize-1;i_++) { pbuf.xy[j,i_] = pbuf.xyrow[i_]; } } } // // Process chunk and advance line pointer // mlpchunkedgradient(network, pbuf.xy, 0, csize, pbuf.batch4buf, pbuf.hpcbuf, ref sgrad.f, false, _params); cstart = cstart+pbuf.chunksize; } hpccores.hpcfinalizechunkedgradient(pbuf, sgrad.g, _params); alglib.smp.ae_shared_pool_recycle(buf, ref pbuf); alglib.smp.ae_shared_pool_recycle(gradbuf, ref sgrad); } /************************************************************************* Serial stub for GPL edition. *************************************************************************/ public static bool _trypexec_mlpgradbatchx(multilayerperceptron network, double[,] densexy, sparse.sparsematrix sparsexy, int datasetsize, int datasettype, int[] idx, int subset0, int subset1, int subsettype, alglib.smp.shared_pool buf, alglib.smp.shared_pool gradbuf, alglib.xparams _params) { return false; } /************************************************************************* Batch gradient calculation for a set of inputs/outputs (natural error function is used) INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - set of inputs/outputs; one sample = one row; first NIn columns contain inputs, next NOut columns - desired outputs. SSize - number of elements in XY Grad - possibly preallocated array. If size of array is smaller than WCount, it will be reallocated. It is recommended to reuse previously allocated array to reduce allocation overhead. OUTPUT PARAMETERS: E - error function, sum-of-squares for regression networks, cross-entropy for classification networks. Grad - gradient of E with respect to weights of network, array[WCount] -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpgradnbatch(multilayerperceptron network, double[,] xy, int ssize, ref double e, ref double[] grad, alglib.xparams _params) { int i = 0; int nin = 0; int nout = 0; int wcount = 0; hpccores.mlpbuffers pbuf = null; e = 0; // // Alloc // mlpproperties(network, ref nin, ref nout, ref wcount, _params); alglib.smp.ae_shared_pool_retrieve(network.buf, ref pbuf); hpccores.hpcpreparechunkedgradient(network.weights, wcount, mlpntotal(network, _params), nin, nout, pbuf, _params); apserv.rvectorsetlengthatleast(ref grad, wcount, _params); for(i=0; i<=wcount-1; i++) { grad[i] = 0; } e = 0; i = 0; while( i<=ssize-1 ) { mlpchunkedgradient(network, xy, i, Math.Min(ssize, i+pbuf.chunksize)-i, pbuf.batch4buf, pbuf.hpcbuf, ref e, true, _params); i = i+pbuf.chunksize; } hpccores.hpcfinalizechunkedgradient(pbuf, grad, _params); alglib.smp.ae_shared_pool_recycle(network.buf, ref pbuf); } /************************************************************************* Batch Hessian calculation (natural error function) using R-algorithm. Internal subroutine. -- ALGLIB -- Copyright 26.01.2008 by Bochkanov Sergey. Hessian calculation based on R-algorithm described in "Fast Exact Multiplication by the Hessian", B. A. Pearlmutter, Neural Computation, 1994. *************************************************************************/ public static void mlphessiannbatch(multilayerperceptron network, double[,] xy, int ssize, ref double e, ref double[] grad, ref double[,] h, alglib.xparams _params) { e = 0; mlphessianbatchinternal(network, xy, ssize, true, ref e, ref grad, ref h, _params); } /************************************************************************* Batch Hessian calculation using R-algorithm. Internal subroutine. -- ALGLIB -- Copyright 26.01.2008 by Bochkanov Sergey. Hessian calculation based on R-algorithm described in "Fast Exact Multiplication by the Hessian", B. A. Pearlmutter, Neural Computation, 1994. *************************************************************************/ public static void mlphessianbatch(multilayerperceptron network, double[,] xy, int ssize, ref double e, ref double[] grad, ref double[,] h, alglib.xparams _params) { e = 0; mlphessianbatchinternal(network, xy, ssize, false, ref e, ref grad, ref h, _params); } /************************************************************************* Internal subroutine, shouldn't be called by user. *************************************************************************/ public static void mlpinternalprocessvector(int[] structinfo, double[] weights, double[] columnmeans, double[] columnsigmas, ref double[] neurons, ref double[] dfdnet, double[] x, ref double[] y, alglib.xparams _params) { int i = 0; int n1 = 0; int n2 = 0; int w1 = 0; int w2 = 0; int ntotal = 0; int nin = 0; int nout = 0; int istart = 0; int offs = 0; double net = 0; double f = 0; double df = 0; double d2f = 0; double mx = 0; bool perr = new bool(); int i_ = 0; int i1_ = 0; // // Read network geometry // nin = structinfo[1]; nout = structinfo[2]; ntotal = structinfo[3]; istart = structinfo[5]; // // Inputs standartisation and putting in the network // for(i=0; i<=nin-1; i++) { if( (double)(columnsigmas[i])!=(double)(0) ) { neurons[i] = (x[i]-columnmeans[i])/columnsigmas[i]; } else { neurons[i] = x[i]-columnmeans[i]; } } // // Process network // for(i=0; i<=ntotal-1; i++) { offs = istart+i*nfieldwidth; if( structinfo[offs+0]>0 || structinfo[offs+0]==-5 ) { // // Activation function // mlpactivationfunction(neurons[structinfo[offs+2]], structinfo[offs+0], ref f, ref df, ref d2f, _params); neurons[i] = f; dfdnet[i] = df; continue; } if( structinfo[offs+0]==0 ) { // // Adaptive summator // n1 = structinfo[offs+2]; n2 = n1+structinfo[offs+1]-1; w1 = structinfo[offs+3]; w2 = w1+structinfo[offs+1]-1; i1_ = (n1)-(w1); net = 0.0; for(i_=w1; i_<=w2;i_++) { net += weights[i_]*neurons[i_+i1_]; } neurons[i] = net; dfdnet[i] = 1.0; apserv.touchint(ref n2, _params); continue; } if( structinfo[offs+0]<0 ) { perr = true; if( structinfo[offs+0]==-2 ) { // // input neuron, left unchanged // perr = false; } if( structinfo[offs+0]==-3 ) { // // "-1" neuron // neurons[i] = -1; perr = false; } if( structinfo[offs+0]==-4 ) { // // "0" neuron // neurons[i] = 0; perr = false; } alglib.ap.assert(!perr, "MLPInternalProcessVector: internal error - unknown neuron type!"); continue; } } // // Extract result // i1_ = (ntotal-nout) - (0); for(i_=0; i_<=nout-1;i_++) { y[i_] = neurons[i_+i1_]; } // // Softmax post-processing or standardisation if needed // alglib.ap.assert(structinfo[6]==0 || structinfo[6]==1, "MLPInternalProcessVector: unknown normalization type!"); if( structinfo[6]==1 ) { // // Softmax // mx = y[0]; for(i=1; i<=nout-1; i++) { mx = Math.Max(mx, y[i]); } net = 0; for(i=0; i<=nout-1; i++) { y[i] = Math.Exp(y[i]-mx); net = net+y[i]; } for(i=0; i<=nout-1; i++) { y[i] = y[i]/net; } } else { // // Standardisation // for(i=0; i<=nout-1; i++) { y[i] = y[i]*columnsigmas[nin+i]+columnmeans[nin+i]; } } } /************************************************************************* Serializer: allocation -- ALGLIB -- Copyright 14.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpalloc(alglib.serializer s, multilayerperceptron network, alglib.xparams _params) { int i = 0; int j = 0; int k = 0; int fkind = 0; double threshold = 0; double v0 = 0; double v1 = 0; int nin = 0; int nout = 0; nin = network.hllayersizes[0]; nout = network.hllayersizes[alglib.ap.len(network.hllayersizes)-1]; s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); apserv.allocintegerarray(s, network.hllayersizes, -1, _params); for(i=1; i<=alglib.ap.len(network.hllayersizes)-1; i++) { for(j=0; j<=network.hllayersizes[i]-1; j++) { mlpgetneuroninfo(network, i, j, ref fkind, ref threshold, _params); s.alloc_entry(); s.alloc_entry(); for(k=0; k<=network.hllayersizes[i-1]-1; k++) { s.alloc_entry(); } } } for(j=0; j<=nin-1; j++) { mlpgetinputscaling(network, j, ref v0, ref v1, _params); s.alloc_entry(); s.alloc_entry(); } for(j=0; j<=nout-1; j++) { mlpgetoutputscaling(network, j, ref v0, ref v1, _params); s.alloc_entry(); s.alloc_entry(); } } /************************************************************************* Serializer: serialization -- ALGLIB -- Copyright 14.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpserialize(alglib.serializer s, multilayerperceptron network, alglib.xparams _params) { int i = 0; int j = 0; int k = 0; int fkind = 0; double threshold = 0; double v0 = 0; double v1 = 0; int nin = 0; int nout = 0; nin = network.hllayersizes[0]; nout = network.hllayersizes[alglib.ap.len(network.hllayersizes)-1]; s.serialize_int(scodes.getmlpserializationcode(_params)); s.serialize_int(mlpfirstversion); s.serialize_bool(mlpissoftmax(network, _params)); apserv.serializeintegerarray(s, network.hllayersizes, -1, _params); for(i=1; i<=alglib.ap.len(network.hllayersizes)-1; i++) { for(j=0; j<=network.hllayersizes[i]-1; j++) { mlpgetneuroninfo(network, i, j, ref fkind, ref threshold, _params); s.serialize_int(fkind); s.serialize_double(threshold); for(k=0; k<=network.hllayersizes[i-1]-1; k++) { s.serialize_double(mlpgetweight(network, i-1, k, i, j, _params)); } } } for(j=0; j<=nin-1; j++) { mlpgetinputscaling(network, j, ref v0, ref v1, _params); s.serialize_double(v0); s.serialize_double(v1); } for(j=0; j<=nout-1; j++) { mlpgetoutputscaling(network, j, ref v0, ref v1, _params); s.serialize_double(v0); s.serialize_double(v1); } } /************************************************************************* Serializer: unserialization -- ALGLIB -- Copyright 14.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpunserialize(alglib.serializer s, multilayerperceptron network, alglib.xparams _params) { int i0 = 0; int i1 = 0; int i = 0; int j = 0; int k = 0; int fkind = 0; double threshold = 0; double v0 = 0; double v1 = 0; int nin = 0; int nout = 0; bool issoftmax = new bool(); int[] layersizes = new int[0]; // // check correctness of header // i0 = s.unserialize_int(); alglib.ap.assert(i0==scodes.getmlpserializationcode(_params), "MLPUnserialize: stream header corrupted"); i1 = s.unserialize_int(); alglib.ap.assert(i1==mlpfirstversion, "MLPUnserialize: stream header corrupted"); // // Create network // issoftmax = s.unserialize_bool(); apserv.unserializeintegerarray(s, ref layersizes, _params); alglib.ap.assert((alglib.ap.len(layersizes)==2 || alglib.ap.len(layersizes)==3) || alglib.ap.len(layersizes)==4, "MLPUnserialize: too many hidden layers!"); nin = layersizes[0]; nout = layersizes[alglib.ap.len(layersizes)-1]; if( alglib.ap.len(layersizes)==2 ) { if( issoftmax ) { mlpcreatec0(layersizes[0], layersizes[1], network, _params); } else { mlpcreate0(layersizes[0], layersizes[1], network, _params); } } if( alglib.ap.len(layersizes)==3 ) { if( issoftmax ) { mlpcreatec1(layersizes[0], layersizes[1], layersizes[2], network, _params); } else { mlpcreate1(layersizes[0], layersizes[1], layersizes[2], network, _params); } } if( alglib.ap.len(layersizes)==4 ) { if( issoftmax ) { mlpcreatec2(layersizes[0], layersizes[1], layersizes[2], layersizes[3], network, _params); } else { mlpcreate2(layersizes[0], layersizes[1], layersizes[2], layersizes[3], network, _params); } } // // Load neurons and weights // for(i=1; i<=alglib.ap.len(layersizes)-1; i++) { for(j=0; j<=layersizes[i]-1; j++) { fkind = s.unserialize_int(); threshold = s.unserialize_double(); mlpsetneuroninfo(network, i, j, fkind, threshold, _params); for(k=0; k<=layersizes[i-1]-1; k++) { v0 = s.unserialize_double(); mlpsetweight(network, i-1, k, i, j, v0, _params); } } } // // Load standartizator // for(j=0; j<=nin-1; j++) { v0 = s.unserialize_double(); v1 = s.unserialize_double(); mlpsetinputscaling(network, j, v0, v1, _params); } for(j=0; j<=nout-1; j++) { v0 = s.unserialize_double(); v1 = s.unserialize_double(); mlpsetoutputscaling(network, j, v0, v1, _params); } } /************************************************************************* Calculation of all types of errors on subset of dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset; one sample = one row; first NIn columns contain inputs, next NOut columns - desired outputs. SetSize - real size of XY, SetSize>=0; Subset - subset of SubsetSize elements, array[SubsetSize]; SubsetSize- number of elements in Subset[] array: * if SubsetSize>0, rows of XY with indices Subset[0]... ...Subset[SubsetSize-1] are processed * if SubsetSize=0, zeros are returned * if SubsetSize<0, entire dataset is processed; Subset[] array is ignored in this case. OUTPUT PARAMETERS: Rep - it contains all type of errors. -- ALGLIB -- Copyright 04.09.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpallerrorssubset(multilayerperceptron network, double[,] xy, int setsize, int[] subset, int subsetsize, modelerrors rep, alglib.xparams _params) { int idx0 = 0; int idx1 = 0; int idxtype = 0; alglib.ap.assert(alglib.ap.rows(xy)>=setsize, "MLPAllErrorsSubset: XY has less than SetSize rows"); if( setsize>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+1, "MLPAllErrorsSubset: XY has less than NIn+1 columns"); } else { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPAllErrorsSubset: XY has less than NIn+NOut columns"); } } if( subsetsize>=0 ) { idx0 = 0; idx1 = subsetsize; idxtype = 1; } else { idx0 = 0; idx1 = setsize; idxtype = 0; } mlpallerrorsx(network, xy, network.dummysxy, setsize, 0, subset, idx0, idx1, idxtype, network.buf, rep, _params); } /************************************************************************* Calculation of all types of errors on subset of dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - network initialized with one of the network creation funcs XY - original dataset given by sparse matrix; one sample = one row; first NIn columns contain inputs, next NOut columns - desired outputs. SetSize - real size of XY, SetSize>=0; Subset - subset of SubsetSize elements, array[SubsetSize]; SubsetSize- number of elements in Subset[] array: * if SubsetSize>0, rows of XY with indices Subset[0]... ...Subset[SubsetSize-1] are processed * if SubsetSize=0, zeros are returned * if SubsetSize<0, entire dataset is processed; Subset[] array is ignored in this case. OUTPUT PARAMETERS: Rep - it contains all type of errors. -- ALGLIB -- Copyright 04.09.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpallerrorssparsesubset(multilayerperceptron network, sparse.sparsematrix xy, int setsize, int[] subset, int subsetsize, modelerrors rep, alglib.xparams _params) { int idx0 = 0; int idx1 = 0; int idxtype = 0; alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPAllErrorsSparseSubset: XY is not in CRS format."); alglib.ap.assert(sparse.sparsegetnrows(xy, _params)>=setsize, "MLPAllErrorsSparseSubset: XY has less than SetSize rows"); if( setsize>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+1, "MLPAllErrorsSparseSubset: XY has less than NIn+1 columns"); } else { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPAllErrorsSparseSubset: XY has less than NIn+NOut columns"); } } if( subsetsize>=0 ) { idx0 = 0; idx1 = subsetsize; idxtype = 1; } else { idx0 = 0; idx1 = setsize; idxtype = 0; } mlpallerrorsx(network, network.dummydxy, xy, setsize, 1, subset, idx0, idx1, idxtype, network.buf, rep, _params); } /************************************************************************* Error of the neural network on subset of dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format; SetSize - real size of XY, SetSize>=0; Subset - subset of SubsetSize elements, array[SubsetSize]; SubsetSize- number of elements in Subset[] array: * if SubsetSize>0, rows of XY with indices Subset[0]... ...Subset[SubsetSize-1] are processed * if SubsetSize=0, zeros are returned * if SubsetSize<0, entire dataset is processed; Subset[] array is ignored in this case. RESULT: sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 04.09.2012 by Bochkanov Sergey *************************************************************************/ public static double mlperrorsubset(multilayerperceptron network, double[,] xy, int setsize, int[] subset, int subsetsize, alglib.xparams _params) { double result = 0; int idx0 = 0; int idx1 = 0; int idxtype = 0; alglib.ap.assert(alglib.ap.rows(xy)>=setsize, "MLPErrorSubset: XY has less than SetSize rows"); if( setsize>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+1, "MLPErrorSubset: XY has less than NIn+1 columns"); } else { alglib.ap.assert(alglib.ap.cols(xy)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPErrorSubset: XY has less than NIn+NOut columns"); } } if( subsetsize>=0 ) { idx0 = 0; idx1 = subsetsize; idxtype = 1; } else { idx0 = 0; idx1 = setsize; idxtype = 0; } mlpallerrorsx(network, xy, network.dummysxy, setsize, 0, subset, idx0, idx1, idxtype, network.buf, network.err, _params); result = math.sqr(network.err.rmserror)*(idx1-idx0)*mlpgetoutputscount(network, _params)/2; return result; } /************************************************************************* Error of the neural network on subset of sparse dataset. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: Network - neural network; XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Sparse matrix must use CRS format for storage. SetSize - real size of XY, SetSize>=0; it is used when SubsetSize<0; Subset - subset of SubsetSize elements, array[SubsetSize]; SubsetSize- number of elements in Subset[] array: * if SubsetSize>0, rows of XY with indices Subset[0]... ...Subset[SubsetSize-1] are processed * if SubsetSize=0, zeros are returned * if SubsetSize<0, entire dataset is processed; Subset[] array is ignored in this case. RESULT: sum-of-squares error, SUM(sqr(y[i]-desired_y[i])/2) DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following dataset format is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 04.09.2012 by Bochkanov Sergey *************************************************************************/ public static double mlperrorsparsesubset(multilayerperceptron network, sparse.sparsematrix xy, int setsize, int[] subset, int subsetsize, alglib.xparams _params) { double result = 0; int idx0 = 0; int idx1 = 0; int idxtype = 0; alglib.ap.assert(sparse.sparseiscrs(xy, _params), "MLPErrorSparseSubset: XY is not in CRS format."); alglib.ap.assert(sparse.sparsegetnrows(xy, _params)>=setsize, "MLPErrorSparseSubset: XY has less than SetSize rows"); if( setsize>0 ) { if( mlpissoftmax(network, _params) ) { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+1, "MLPErrorSparseSubset: XY has less than NIn+1 columns"); } else { alglib.ap.assert(sparse.sparsegetncols(xy, _params)>=mlpgetinputscount(network, _params)+mlpgetoutputscount(network, _params), "MLPErrorSparseSubset: XY has less than NIn+NOut columns"); } } if( subsetsize>=0 ) { idx0 = 0; idx1 = subsetsize; idxtype = 1; } else { idx0 = 0; idx1 = setsize; idxtype = 0; } mlpallerrorsx(network, network.dummydxy, xy, setsize, 1, subset, idx0, idx1, idxtype, network.buf, network.err, _params); result = math.sqr(network.err.rmserror)*(idx1-idx0)*mlpgetoutputscount(network, _params)/2; return result; } /************************************************************************* Calculation of all types of errors at once for a subset or full dataset, which can be represented in different formats. THIS INTERNAL FUNCTION IS NOT INTENDED TO BE USED BY ALGLIB USERS! -- ALGLIB -- Copyright 26.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpallerrorsx(multilayerperceptron network, double[,] densexy, sparse.sparsematrix sparsexy, int datasetsize, int datasettype, int[] idx, int subset0, int subset1, int subsettype, alglib.smp.shared_pool buf, modelerrors rep, alglib.xparams _params) { int nin = 0; int nout = 0; int wcount = 0; int rowsize = 0; bool iscls = new bool(); int srcidx = 0; int cstart = 0; int csize = 0; int j = 0; hpccores.mlpbuffers pbuf = null; int len0 = 0; int len1 = 0; modelerrors rep0 = new modelerrors(); modelerrors rep1 = new modelerrors(); double problemcost = 0; int i_ = 0; int i1_ = 0; alglib.ap.assert(datasetsize>=0, "MLPAllErrorsX: SetSize<0"); alglib.ap.assert(datasettype==0 || datasettype==1, "MLPAllErrorsX: DatasetType is incorrect"); alglib.ap.assert(subsettype==0 || subsettype==1, "MLPAllErrorsX: SubsetType is incorrect"); // // Determine network properties // mlpproperties(network, ref nin, ref nout, ref wcount, _params); iscls = mlpissoftmax(network, _params); // // Split problem. // // Splitting problem allows us to reduce effect of single-precision // arithmetics (SSE-optimized version of MLPChunkedProcess uses single // precision internally, but converts them to double precision after // results are exported from HPC buffer to network). Small batches are // calculated in single precision, results are aggregated in double // precision, and it allows us to avoid accumulation of errors when // we process very large batches (tens of thousands of items). // // NOTE: it is important to use real arithmetics for ProblemCost // because ProblemCost may be larger than MAXINT. // problemcost = subset1-subset0; problemcost = problemcost*wcount*2; if( (double)(problemcost)>=(double)(apserv.smpactivationlevel(_params)) && subset1-subset0>=2*microbatchsize ) { if( _trypexec_mlpallerrorsx(network,densexy,sparsexy,datasetsize,datasettype,idx,subset0,subset1,subsettype,buf,rep, _params) ) { return; } } if( subset1-subset0>=2*microbatchsize && (double)(problemcost)>(double)(apserv.spawnlevel(_params)) ) { apserv.splitlength(subset1-subset0, microbatchsize, ref len0, ref len1, _params); mlpallerrorsx(network, densexy, sparsexy, datasetsize, datasettype, idx, subset0, subset0+len0, subsettype, buf, rep0, _params); mlpallerrorsx(network, densexy, sparsexy, datasetsize, datasettype, idx, subset0+len0, subset1, subsettype, buf, rep1, _params); rep.relclserror = (len0*rep0.relclserror+len1*rep1.relclserror)/(len0+len1); rep.avgce = (len0*rep0.avgce+len1*rep1.avgce)/(len0+len1); rep.rmserror = Math.Sqrt((len0*math.sqr(rep0.rmserror)+len1*math.sqr(rep1.rmserror))/(len0+len1)); rep.avgerror = (len0*rep0.avgerror+len1*rep1.avgerror)/(len0+len1); rep.avgrelerror = (len0*rep0.avgrelerror+len1*rep1.avgrelerror)/(len0+len1); return; } // // Retrieve and prepare // alglib.smp.ae_shared_pool_retrieve(buf, ref pbuf); if( iscls ) { rowsize = nin+1; bdss.dserrallocate(nout, ref pbuf.tmp0, _params); } else { rowsize = nin+nout; bdss.dserrallocate(-nout, ref pbuf.tmp0, _params); } // // Processing // hpccores.hpcpreparechunkedgradient(network.weights, wcount, mlpntotal(network, _params), nin, nout, pbuf, _params); cstart = subset0; while( cstart=0, "MLPAllErrorsX: internal error"); if( datasettype==0 ) { for(i_=0; i_<=rowsize-1;i_++) { pbuf.xy[j,i_] = densexy[srcidx,i_]; } } if( datasettype==1 ) { sparse.sparsegetrow(sparsexy, srcidx, ref pbuf.xyrow, _params); for(i_=0; i_<=rowsize-1;i_++) { pbuf.xy[j,i_] = pbuf.xyrow[i_]; } } } // // Unpack XY and process (temporary code, to be replaced by chunked processing) // for(j=0; j<=csize-1; j++) { for(i_=0; i_<=rowsize-1;i_++) { pbuf.xy2[j,i_] = pbuf.xy[j,i_]; } } mlpchunkedprocess(network, pbuf.xy2, 0, csize, pbuf.batch4buf, pbuf.hpcbuf, _params); for(j=0; j<=csize-1; j++) { for(i_=0; i_<=nin-1;i_++) { pbuf.x[i_] = pbuf.xy2[j,i_]; } i1_ = (nin) - (0); for(i_=0; i_<=nout-1;i_++) { pbuf.y[i_] = pbuf.xy2[j,i_+i1_]; } if( iscls ) { pbuf.desiredy[0] = pbuf.xy[j,nin]; } else { i1_ = (nin) - (0); for(i_=0; i_<=nout-1;i_++) { pbuf.desiredy[i_] = pbuf.xy[j,i_+i1_]; } } bdss.dserraccumulate(ref pbuf.tmp0, pbuf.y, pbuf.desiredy, _params); } // // Process chunk and advance line pointer // cstart = cstart+pbuf.chunksize; } bdss.dserrfinish(ref pbuf.tmp0, _params); rep.relclserror = pbuf.tmp0[0]; rep.avgce = pbuf.tmp0[1]/Math.Log(2); rep.rmserror = pbuf.tmp0[2]; rep.avgerror = pbuf.tmp0[3]; rep.avgrelerror = pbuf.tmp0[4]; // // Recycle // alglib.smp.ae_shared_pool_recycle(buf, ref pbuf); } /************************************************************************* Serial stub for GPL edition. *************************************************************************/ public static bool _trypexec_mlpallerrorsx(multilayerperceptron network, double[,] densexy, sparse.sparsematrix sparsexy, int datasetsize, int datasettype, int[] idx, int subset0, int subset1, int subsettype, alglib.smp.shared_pool buf, modelerrors rep, alglib.xparams _params) { return false; } /************************************************************************* Internal subroutine: adding new input layer to network *************************************************************************/ private static void addinputlayer(int ncount, ref int[] lsizes, ref int[] ltypes, ref int[] lconnfirst, ref int[] lconnlast, ref int lastproc, alglib.xparams _params) { lsizes[0] = ncount; ltypes[0] = -2; lconnfirst[0] = 0; lconnlast[0] = 0; lastproc = 0; } /************************************************************************* Internal subroutine: adding new summator layer to network *************************************************************************/ private static void addbiasedsummatorlayer(int ncount, ref int[] lsizes, ref int[] ltypes, ref int[] lconnfirst, ref int[] lconnlast, ref int lastproc, alglib.xparams _params) { lsizes[lastproc+1] = 1; ltypes[lastproc+1] = -3; lconnfirst[lastproc+1] = 0; lconnlast[lastproc+1] = 0; lsizes[lastproc+2] = ncount; ltypes[lastproc+2] = 0; lconnfirst[lastproc+2] = lastproc; lconnlast[lastproc+2] = lastproc+1; lastproc = lastproc+2; } /************************************************************************* Internal subroutine: adding new summator layer to network *************************************************************************/ private static void addactivationlayer(int functype, ref int[] lsizes, ref int[] ltypes, ref int[] lconnfirst, ref int[] lconnlast, ref int lastproc, alglib.xparams _params) { alglib.ap.assert(functype>0 || functype==-5, "AddActivationLayer: incorrect function type"); lsizes[lastproc+1] = lsizes[lastproc]; ltypes[lastproc+1] = functype; lconnfirst[lastproc+1] = lastproc; lconnlast[lastproc+1] = lastproc; lastproc = lastproc+1; } /************************************************************************* Internal subroutine: adding new zero layer to network *************************************************************************/ private static void addzerolayer(ref int[] lsizes, ref int[] ltypes, ref int[] lconnfirst, ref int[] lconnlast, ref int lastproc, alglib.xparams _params) { lsizes[lastproc+1] = 1; ltypes[lastproc+1] = -4; lconnfirst[lastproc+1] = 0; lconnlast[lastproc+1] = 0; lastproc = lastproc+1; } /************************************************************************* This routine adds input layer to the high-level description of the network. It modifies Network.HLConnections and Network.HLNeurons and assumes that these arrays have enough place to store data. It accepts following parameters: Network - network ConnIdx - index of the first free entry in the HLConnections NeuroIdx - index of the first free entry in the HLNeurons StructInfoIdx- index of the first entry in the low level description of the current layer (in the StructInfo array) NIn - number of inputs It modified Network and indices. *************************************************************************/ private static void hladdinputlayer(multilayerperceptron network, ref int connidx, ref int neuroidx, ref int structinfoidx, int nin, alglib.xparams _params) { int i = 0; int offs = 0; offs = hlnfieldwidth*neuroidx; for(i=0; i<=nin-1; i++) { network.hlneurons[offs+0] = 0; network.hlneurons[offs+1] = i; network.hlneurons[offs+2] = -1; network.hlneurons[offs+3] = -1; offs = offs+hlnfieldwidth; } neuroidx = neuroidx+nin; structinfoidx = structinfoidx+nin; } /************************************************************************* This routine adds output layer to the high-level description of the network. It modifies Network.HLConnections and Network.HLNeurons and assumes that these arrays have enough place to store data. It accepts following parameters: Network - network ConnIdx - index of the first free entry in the HLConnections NeuroIdx - index of the first free entry in the HLNeurons StructInfoIdx- index of the first entry in the low level description of the current layer (in the StructInfo array) WeightsIdx - index of the first entry in the Weights array which corresponds to the current layer K - current layer index NPrev - number of neurons in the previous layer NOut - number of outputs IsCls - is it classifier network? IsLinear - is it network with linear output? It modified Network and ConnIdx/NeuroIdx/StructInfoIdx/WeightsIdx. *************************************************************************/ private static void hladdoutputlayer(multilayerperceptron network, ref int connidx, ref int neuroidx, ref int structinfoidx, ref int weightsidx, int k, int nprev, int nout, bool iscls, bool islinearout, alglib.xparams _params) { int i = 0; int j = 0; int neurooffs = 0; int connoffs = 0; alglib.ap.assert((iscls && islinearout) || !iscls, "HLAddOutputLayer: internal error"); neurooffs = hlnfieldwidth*neuroidx; connoffs = hlconnfieldwidth*connidx; if( !iscls ) { // // Regression network // for(i=0; i<=nout-1; i++) { network.hlneurons[neurooffs+0] = k; network.hlneurons[neurooffs+1] = i; network.hlneurons[neurooffs+2] = structinfoidx+1+nout+i; network.hlneurons[neurooffs+3] = weightsidx+nprev+(nprev+1)*i; neurooffs = neurooffs+hlnfieldwidth; } for(i=0; i<=nprev-1; i++) { for(j=0; j<=nout-1; j++) { network.hlconnections[connoffs+0] = k-1; network.hlconnections[connoffs+1] = i; network.hlconnections[connoffs+2] = k; network.hlconnections[connoffs+3] = j; network.hlconnections[connoffs+4] = weightsidx+i+j*(nprev+1); connoffs = connoffs+hlconnfieldwidth; } } connidx = connidx+nprev*nout; neuroidx = neuroidx+nout; structinfoidx = structinfoidx+2*nout+1; weightsidx = weightsidx+nout*(nprev+1); } else { // // Classification network // for(i=0; i<=nout-2; i++) { network.hlneurons[neurooffs+0] = k; network.hlneurons[neurooffs+1] = i; network.hlneurons[neurooffs+2] = -1; network.hlneurons[neurooffs+3] = weightsidx+nprev+(nprev+1)*i; neurooffs = neurooffs+hlnfieldwidth; } network.hlneurons[neurooffs+0] = k; network.hlneurons[neurooffs+1] = i; network.hlneurons[neurooffs+2] = -1; network.hlneurons[neurooffs+3] = -1; for(i=0; i<=nprev-1; i++) { for(j=0; j<=nout-2; j++) { network.hlconnections[connoffs+0] = k-1; network.hlconnections[connoffs+1] = i; network.hlconnections[connoffs+2] = k; network.hlconnections[connoffs+3] = j; network.hlconnections[connoffs+4] = weightsidx+i+j*(nprev+1); connoffs = connoffs+hlconnfieldwidth; } } connidx = connidx+nprev*(nout-1); neuroidx = neuroidx+nout; structinfoidx = structinfoidx+nout+2; weightsidx = weightsidx+(nout-1)*(nprev+1); } } /************************************************************************* This routine adds hidden layer to the high-level description of the network. It modifies Network.HLConnections and Network.HLNeurons and assumes that these arrays have enough place to store data. It accepts following parameters: Network - network ConnIdx - index of the first free entry in the HLConnections NeuroIdx - index of the first free entry in the HLNeurons StructInfoIdx- index of the first entry in the low level description of the current layer (in the StructInfo array) WeightsIdx - index of the first entry in the Weights array which corresponds to the current layer K - current layer index NPrev - number of neurons in the previous layer NCur - number of neurons in the current layer It modified Network and ConnIdx/NeuroIdx/StructInfoIdx/WeightsIdx. *************************************************************************/ private static void hladdhiddenlayer(multilayerperceptron network, ref int connidx, ref int neuroidx, ref int structinfoidx, ref int weightsidx, int k, int nprev, int ncur, alglib.xparams _params) { int i = 0; int j = 0; int neurooffs = 0; int connoffs = 0; neurooffs = hlnfieldwidth*neuroidx; connoffs = hlconnfieldwidth*connidx; for(i=0; i<=ncur-1; i++) { network.hlneurons[neurooffs+0] = k; network.hlneurons[neurooffs+1] = i; network.hlneurons[neurooffs+2] = structinfoidx+1+ncur+i; network.hlneurons[neurooffs+3] = weightsidx+nprev+(nprev+1)*i; neurooffs = neurooffs+hlnfieldwidth; } for(i=0; i<=nprev-1; i++) { for(j=0; j<=ncur-1; j++) { network.hlconnections[connoffs+0] = k-1; network.hlconnections[connoffs+1] = i; network.hlconnections[connoffs+2] = k; network.hlconnections[connoffs+3] = j; network.hlconnections[connoffs+4] = weightsidx+i+j*(nprev+1); connoffs = connoffs+hlconnfieldwidth; } } connidx = connidx+nprev*ncur; neuroidx = neuroidx+ncur; structinfoidx = structinfoidx+2*ncur+1; weightsidx = weightsidx+ncur*(nprev+1); } /************************************************************************* This function fills high level information about network created using internal MLPCreate() function. This function does NOT examine StructInfo for low level information, it just expects that network has following structure: input neuron \ ... | input layer input neuron / "-1" neuron \ biased summator | ... | biased summator | hidden layer(s), if there are exists any activation function | ... | activation function / "-1" neuron \ biased summator | output layer: ... | biased summator | * we have NOut summators/activators for regression networks activation function | * we have only NOut-1 summators and no activators for classifiers ... | * we have "0" neuron only when we have classifier activation function | "0" neuron / -- ALGLIB -- Copyright 30.03.2008 by Bochkanov Sergey *************************************************************************/ private static void fillhighlevelinformation(multilayerperceptron network, int nin, int nhid1, int nhid2, int nout, bool iscls, bool islinearout, alglib.xparams _params) { int idxweights = 0; int idxstruct = 0; int idxneuro = 0; int idxconn = 0; alglib.ap.assert((iscls && islinearout) || !iscls, "FillHighLevelInformation: internal error"); // // Preparations common to all types of networks // idxweights = 0; idxneuro = 0; idxstruct = 0; idxconn = 0; network.hlnetworktype = 0; // // network without hidden layers // if( nhid1==0 ) { network.hllayersizes = new int[2]; network.hllayersizes[0] = nin; network.hllayersizes[1] = nout; if( !iscls ) { network.hlconnections = new int[hlconnfieldwidth*nin*nout]; network.hlneurons = new int[hlnfieldwidth*(nin+nout)]; network.hlnormtype = 0; } else { network.hlconnections = new int[hlconnfieldwidth*nin*(nout-1)]; network.hlneurons = new int[hlnfieldwidth*(nin+nout)]; network.hlnormtype = 1; } hladdinputlayer(network, ref idxconn, ref idxneuro, ref idxstruct, nin, _params); hladdoutputlayer(network, ref idxconn, ref idxneuro, ref idxstruct, ref idxweights, 1, nin, nout, iscls, islinearout, _params); return; } // // network with one hidden layers // if( nhid2==0 ) { network.hllayersizes = new int[3]; network.hllayersizes[0] = nin; network.hllayersizes[1] = nhid1; network.hllayersizes[2] = nout; if( !iscls ) { network.hlconnections = new int[hlconnfieldwidth*(nin*nhid1+nhid1*nout)]; network.hlneurons = new int[hlnfieldwidth*(nin+nhid1+nout)]; network.hlnormtype = 0; } else { network.hlconnections = new int[hlconnfieldwidth*(nin*nhid1+nhid1*(nout-1))]; network.hlneurons = new int[hlnfieldwidth*(nin+nhid1+nout)]; network.hlnormtype = 1; } hladdinputlayer(network, ref idxconn, ref idxneuro, ref idxstruct, nin, _params); hladdhiddenlayer(network, ref idxconn, ref idxneuro, ref idxstruct, ref idxweights, 1, nin, nhid1, _params); hladdoutputlayer(network, ref idxconn, ref idxneuro, ref idxstruct, ref idxweights, 2, nhid1, nout, iscls, islinearout, _params); return; } // // Two hidden layers // network.hllayersizes = new int[4]; network.hllayersizes[0] = nin; network.hllayersizes[1] = nhid1; network.hllayersizes[2] = nhid2; network.hllayersizes[3] = nout; if( !iscls ) { network.hlconnections = new int[hlconnfieldwidth*(nin*nhid1+nhid1*nhid2+nhid2*nout)]; network.hlneurons = new int[hlnfieldwidth*(nin+nhid1+nhid2+nout)]; network.hlnormtype = 0; } else { network.hlconnections = new int[hlconnfieldwidth*(nin*nhid1+nhid1*nhid2+nhid2*(nout-1))]; network.hlneurons = new int[hlnfieldwidth*(nin+nhid1+nhid2+nout)]; network.hlnormtype = 1; } hladdinputlayer(network, ref idxconn, ref idxneuro, ref idxstruct, nin, _params); hladdhiddenlayer(network, ref idxconn, ref idxneuro, ref idxstruct, ref idxweights, 1, nin, nhid1, _params); hladdhiddenlayer(network, ref idxconn, ref idxneuro, ref idxstruct, ref idxweights, 2, nhid1, nhid2, _params); hladdoutputlayer(network, ref idxconn, ref idxneuro, ref idxstruct, ref idxweights, 3, nhid2, nout, iscls, islinearout, _params); } /************************************************************************* Internal subroutine. -- ALGLIB -- Copyright 04.11.2007 by Bochkanov Sergey *************************************************************************/ private static void mlpcreate(int nin, int nout, int[] lsizes, int[] ltypes, int[] lconnfirst, int[] lconnlast, int layerscount, bool isclsnet, multilayerperceptron network, alglib.xparams _params) { int i = 0; int j = 0; int ssize = 0; int ntotal = 0; int wcount = 0; int offs = 0; int nprocessed = 0; int wallocated = 0; int[] localtemp = new int[0]; int[] lnfirst = new int[0]; int[] lnsyn = new int[0]; hpccores.mlpbuffers buf = new hpccores.mlpbuffers(); smlpgrad sgrad = new smlpgrad(); // // Check // alglib.ap.assert(layerscount>0, "MLPCreate: wrong parameters!"); alglib.ap.assert(ltypes[0]==-2, "MLPCreate: wrong LTypes[0] (must be -2)!"); for(i=0; i<=layerscount-1; i++) { alglib.ap.assert(lsizes[i]>0, "MLPCreate: wrong LSizes!"); alglib.ap.assert(lconnfirst[i]>=0 && (lconnfirst[i]=lconnfirst[i] && (lconnlast[i]=0 || ltypes[i]==-5 ) { lnsyn[i] = 0; for(j=lconnfirst[i]; j<=lconnlast[i]; j++) { lnsyn[i] = lnsyn[i]+lsizes[j]; } } else { if( (ltypes[i]==-2 || ltypes[i]==-3) || ltypes[i]==-4 ) { lnsyn[i] = 0; } } alglib.ap.assert(lnsyn[i]>=0, "MLPCreate: internal error #0!"); // // Other info // lnfirst[i] = ntotal; ntotal = ntotal+lsizes[i]; if( ltypes[i]==0 ) { wcount = wcount+lnsyn[i]*lsizes[i]; } } ssize = 7+ntotal*nfieldwidth; // // Allocate // network.structinfo = new int[ssize-1+1]; network.weights = new double[wcount-1+1]; if( isclsnet ) { network.columnmeans = new double[nin-1+1]; network.columnsigmas = new double[nin-1+1]; } else { network.columnmeans = new double[nin+nout-1+1]; network.columnsigmas = new double[nin+nout-1+1]; } network.neurons = new double[ntotal-1+1]; network.nwbuf = new double[Math.Max(wcount, 2*nout)-1+1]; network.integerbuf = new int[3+1]; network.dfdnet = new double[ntotal-1+1]; network.x = new double[nin-1+1]; network.y = new double[nout-1+1]; network.derror = new double[ntotal-1+1]; // // Fill structure: // * first, fill by dummy values to avoid spurious reports by Valgrind // * then fill global info header // for(i=0; i<=ssize-1; i++) { network.structinfo[i] = -999999; } network.structinfo[0] = ssize; network.structinfo[1] = nin; network.structinfo[2] = nout; network.structinfo[3] = ntotal; network.structinfo[4] = wcount; network.structinfo[5] = 7; if( isclsnet ) { network.structinfo[6] = 1; } else { network.structinfo[6] = 0; } // // Fill structure: neuron connections // nprocessed = 0; wallocated = 0; for(i=0; i<=layerscount-1; i++) { for(j=0; j<=lsizes[i]-1; j++) { offs = network.structinfo[5]+nprocessed*nfieldwidth; network.structinfo[offs+0] = ltypes[i]; if( ltypes[i]==0 ) { // // Adaptive summator: // * connections with weights to previous neurons // network.structinfo[offs+1] = lnsyn[i]; network.structinfo[offs+2] = lnfirst[lconnfirst[i]]; network.structinfo[offs+3] = wallocated; wallocated = wallocated+lnsyn[i]; nprocessed = nprocessed+1; } if( ltypes[i]>0 || ltypes[i]==-5 ) { // // Activation layer: // * each neuron connected to one (only one) of previous neurons. // * no weights // network.structinfo[offs+1] = 1; network.structinfo[offs+2] = lnfirst[lconnfirst[i]]+j; network.structinfo[offs+3] = -1; nprocessed = nprocessed+1; } if( (ltypes[i]==-2 || ltypes[i]==-3) || ltypes[i]==-4 ) { nprocessed = nprocessed+1; } } } alglib.ap.assert(wallocated==wcount, "MLPCreate: internal error #1!"); alglib.ap.assert(nprocessed==ntotal, "MLPCreate: internal error #2!"); // // Fill weights by small random values // Initialize means and sigmas // for(i=0; i<=nin-1; i++) { network.columnmeans[i] = 0; network.columnsigmas[i] = 1; } if( !isclsnet ) { for(i=0; i<=nout-1; i++) { network.columnmeans[nin+i] = 0; network.columnsigmas[nin+i] = 1; } } mlprandomize(network, _params); // // Seed buffers // alglib.smp.ae_shared_pool_set_seed(network.buf, buf); sgrad.g = new double[wcount]; sgrad.f = 0.0; for(i=0; i<=wcount-1; i++) { sgrad.g[i] = 0.0; } alglib.smp.ae_shared_pool_set_seed(network.gradbuf, sgrad); } /************************************************************************* Internal subroutine for Hessian calculation. WARNING! Unspeakable math far beyong human capabilities :) *************************************************************************/ private static void mlphessianbatchinternal(multilayerperceptron network, double[,] xy, int ssize, bool naturalerr, ref double e, ref double[] grad, ref double[,] h, alglib.xparams _params) { int nin = 0; int nout = 0; int wcount = 0; int ntotal = 0; int istart = 0; int i = 0; int j = 0; int k = 0; int kl = 0; int offs = 0; int n1 = 0; int n2 = 0; int w1 = 0; int w2 = 0; double s = 0; double t = 0; double v = 0; double et = 0; bool bflag = new bool(); double f = 0; double df = 0; double d2f = 0; double deidyj = 0; double mx = 0; double q = 0; double z = 0; double s2 = 0; double expi = 0; double expj = 0; double[] x = new double[0]; double[] desiredy = new double[0]; double[] gt = new double[0]; double[] zeros = new double[0]; double[,] rx = new double[0,0]; double[,] ry = new double[0,0]; double[,] rdx = new double[0,0]; double[,] rdy = new double[0,0]; int i_ = 0; int i1_ = 0; e = 0; mlpproperties(network, ref nin, ref nout, ref wcount, _params); ntotal = network.structinfo[3]; istart = network.structinfo[5]; // // Prepare // x = new double[nin-1+1]; desiredy = new double[nout-1+1]; zeros = new double[wcount-1+1]; gt = new double[wcount-1+1]; rx = new double[ntotal+nout-1+1, wcount-1+1]; ry = new double[ntotal+nout-1+1, wcount-1+1]; rdx = new double[ntotal+nout-1+1, wcount-1+1]; rdy = new double[ntotal+nout-1+1, wcount-1+1]; e = 0; for(i=0; i<=wcount-1; i++) { zeros[i] = 0; } for(i_=0; i_<=wcount-1;i_++) { grad[i_] = zeros[i_]; } for(i=0; i<=wcount-1; i++) { for(i_=0; i_<=wcount-1;i_++) { h[i,i_] = zeros[i_]; } } // // Process // for(k=0; k<=ssize-1; k++) { // // Process vector with MLPGradN. // Now Neurons, DFDNET and DError contains results of the last run. // for(i_=0; i_<=nin-1;i_++) { x[i_] = xy[k,i_]; } if( mlpissoftmax(network, _params) ) { // // class labels outputs // kl = (int)Math.Round(xy[k,nin]); for(i=0; i<=nout-1; i++) { if( i==kl ) { desiredy[i] = 1; } else { desiredy[i] = 0; } } } else { // // real outputs // i1_ = (nin) - (0); for(i_=0; i_<=nout-1;i_++) { desiredy[i_] = xy[k,i_+i1_]; } } if( naturalerr ) { mlpgradn(network, x, desiredy, ref et, ref gt, _params); } else { mlpgrad(network, x, desiredy, ref et, ref gt, _params); } // // grad, error // e = e+et; for(i_=0; i_<=wcount-1;i_++) { grad[i_] = grad[i_] + gt[i_]; } // // Hessian. // Forward pass of the R-algorithm // for(i=0; i<=ntotal-1; i++) { offs = istart+i*nfieldwidth; for(i_=0; i_<=wcount-1;i_++) { rx[i,i_] = zeros[i_]; } for(i_=0; i_<=wcount-1;i_++) { ry[i,i_] = zeros[i_]; } if( network.structinfo[offs+0]>0 || network.structinfo[offs+0]==-5 ) { // // Activation function // n1 = network.structinfo[offs+2]; for(i_=0; i_<=wcount-1;i_++) { rx[i,i_] = ry[n1,i_]; } v = network.dfdnet[i]; for(i_=0; i_<=wcount-1;i_++) { ry[i,i_] = v*rx[i,i_]; } continue; } if( network.structinfo[offs+0]==0 ) { // // Adaptive summator // n1 = network.structinfo[offs+2]; n2 = n1+network.structinfo[offs+1]-1; w1 = network.structinfo[offs+3]; w2 = w1+network.structinfo[offs+1]-1; for(j=n1; j<=n2; j++) { v = network.weights[w1+j-n1]; for(i_=0; i_<=wcount-1;i_++) { rx[i,i_] = rx[i,i_] + v*ry[j,i_]; } rx[i,w1+j-n1] = rx[i,w1+j-n1]+network.neurons[j]; } for(i_=0; i_<=wcount-1;i_++) { ry[i,i_] = rx[i,i_]; } continue; } if( network.structinfo[offs+0]<0 ) { bflag = true; if( network.structinfo[offs+0]==-2 ) { // // input neuron, left unchanged // bflag = false; } if( network.structinfo[offs+0]==-3 ) { // // "-1" neuron, left unchanged // bflag = false; } if( network.structinfo[offs+0]==-4 ) { // // "0" neuron, left unchanged // bflag = false; } alglib.ap.assert(!bflag, "MLPHessianNBatch: internal error - unknown neuron type!"); continue; } } // // Hessian. Backward pass of the R-algorithm. // // Stage 1. Initialize RDY // for(i=0; i<=ntotal+nout-1; i++) { for(i_=0; i_<=wcount-1;i_++) { rdy[i,i_] = zeros[i_]; } } if( network.structinfo[6]==0 ) { // // Standardisation. // // In context of the Hessian calculation standardisation // is considered as additional layer with weightless // activation function: // // F(NET) := Sigma*NET // // So we add one more layer to forward pass, and // make forward/backward pass through this layer. // for(i=0; i<=nout-1; i++) { n1 = ntotal-nout+i; n2 = ntotal+i; // // Forward pass from N1 to N2 // for(i_=0; i_<=wcount-1;i_++) { rx[n2,i_] = ry[n1,i_]; } v = network.columnsigmas[nin+i]; for(i_=0; i_<=wcount-1;i_++) { ry[n2,i_] = v*rx[n2,i_]; } // // Initialization of RDY // for(i_=0; i_<=wcount-1;i_++) { rdy[n2,i_] = ry[n2,i_]; } // // Backward pass from N2 to N1: // 1. Calculate R(dE/dX). // 2. No R(dE/dWij) is needed since weight of activation neuron // is fixed to 1. So we can update R(dE/dY) for // the connected neuron (note that Vij=0, Wij=1) // df = network.columnsigmas[nin+i]; for(i_=0; i_<=wcount-1;i_++) { rdx[n2,i_] = df*rdy[n2,i_]; } for(i_=0; i_<=wcount-1;i_++) { rdy[n1,i_] = rdy[n1,i_] + rdx[n2,i_]; } } } else { // // Softmax. // // Initialize RDY using generalized expression for ei'(yi) // (see expression (9) from p. 5 of "Fast Exact Multiplication by the Hessian"). // // When we are working with softmax network, generalized // expression for ei'(yi) is used because softmax // normalization leads to ei, which depends on all y's // if( naturalerr ) { // // softmax + cross-entropy. // We have: // // S = sum(exp(yk)), // ei = sum(trn)*exp(yi)/S-trn_i // // j=i: d(ei)/d(yj) = T*exp(yi)*(S-exp(yi))/S^2 // j<>i: d(ei)/d(yj) = -T*exp(yi)*exp(yj)/S^2 // t = 0; for(i=0; i<=nout-1; i++) { t = t+desiredy[i]; } mx = network.neurons[ntotal-nout]; for(i=0; i<=nout-1; i++) { mx = Math.Max(mx, network.neurons[ntotal-nout+i]); } s = 0; for(i=0; i<=nout-1; i++) { network.nwbuf[i] = Math.Exp(network.neurons[ntotal-nout+i]-mx); s = s+network.nwbuf[i]; } for(i=0; i<=nout-1; i++) { for(j=0; j<=nout-1; j++) { if( j==i ) { deidyj = t*network.nwbuf[i]*(s-network.nwbuf[i])/math.sqr(s); for(i_=0; i_<=wcount-1;i_++) { rdy[ntotal-nout+i,i_] = rdy[ntotal-nout+i,i_] + deidyj*ry[ntotal-nout+i,i_]; } } else { deidyj = -(t*network.nwbuf[i]*network.nwbuf[j]/math.sqr(s)); for(i_=0; i_<=wcount-1;i_++) { rdy[ntotal-nout+i,i_] = rdy[ntotal-nout+i,i_] + deidyj*ry[ntotal-nout+j,i_]; } } } } } else { // // For a softmax + squared error we have expression // far beyond human imagination so we dont even try // to comment on it. Just enjoy the code... // // P.S. That's why "natural error" is called "natural" - // compact beatiful expressions, fast code.... // mx = network.neurons[ntotal-nout]; for(i=0; i<=nout-1; i++) { mx = Math.Max(mx, network.neurons[ntotal-nout+i]); } s = 0; s2 = 0; for(i=0; i<=nout-1; i++) { network.nwbuf[i] = Math.Exp(network.neurons[ntotal-nout+i]-mx); s = s+network.nwbuf[i]; s2 = s2+math.sqr(network.nwbuf[i]); } q = 0; for(i=0; i<=nout-1; i++) { q = q+(network.y[i]-desiredy[i])*network.nwbuf[i]; } for(i=0; i<=nout-1; i++) { z = -q+(network.y[i]-desiredy[i])*s; expi = network.nwbuf[i]; for(j=0; j<=nout-1; j++) { expj = network.nwbuf[j]; if( j==i ) { deidyj = expi/math.sqr(s)*((z+expi)*(s-2*expi)/s+expi*s2/math.sqr(s)); } else { deidyj = expi*expj/math.sqr(s)*(s2/math.sqr(s)-2*z/s-(expi+expj)/s+(network.y[i]-desiredy[i])-(network.y[j]-desiredy[j])); } for(i_=0; i_<=wcount-1;i_++) { rdy[ntotal-nout+i,i_] = rdy[ntotal-nout+i,i_] + deidyj*ry[ntotal-nout+j,i_]; } } } } } // // Hessian. Backward pass of the R-algorithm // // Stage 2. Process. // for(i=ntotal-1; i>=0; i--) { // // Possible variants: // 1. Activation function // 2. Adaptive summator // 3. Special neuron // offs = istart+i*nfieldwidth; if( network.structinfo[offs+0]>0 || network.structinfo[offs+0]==-5 ) { n1 = network.structinfo[offs+2]; // // First, calculate R(dE/dX). // mlpactivationfunction(network.neurons[n1], network.structinfo[offs+0], ref f, ref df, ref d2f, _params); v = d2f*network.derror[i]; for(i_=0; i_<=wcount-1;i_++) { rdx[i,i_] = df*rdy[i,i_]; } for(i_=0; i_<=wcount-1;i_++) { rdx[i,i_] = rdx[i,i_] + v*rx[i,i_]; } // // No R(dE/dWij) is needed since weight of activation neuron // is fixed to 1. // // So we can update R(dE/dY) for the connected neuron. // (note that Vij=0, Wij=1) // for(i_=0; i_<=wcount-1;i_++) { rdy[n1,i_] = rdy[n1,i_] + rdx[i,i_]; } continue; } if( network.structinfo[offs+0]==0 ) { // // Adaptive summator // n1 = network.structinfo[offs+2]; n2 = n1+network.structinfo[offs+1]-1; w1 = network.structinfo[offs+3]; w2 = w1+network.structinfo[offs+1]-1; // // First, calculate R(dE/dX). // for(i_=0; i_<=wcount-1;i_++) { rdx[i,i_] = rdy[i,i_]; } // // Then, calculate R(dE/dWij) // for(j=w1; j<=w2; j++) { v = network.neurons[n1+j-w1]; for(i_=0; i_<=wcount-1;i_++) { h[j,i_] = h[j,i_] + v*rdx[i,i_]; } v = network.derror[i]; for(i_=0; i_<=wcount-1;i_++) { h[j,i_] = h[j,i_] + v*ry[n1+j-w1,i_]; } } // // And finally, update R(dE/dY) for connected neurons. // for(j=w1; j<=w2; j++) { v = network.weights[j]; for(i_=0; i_<=wcount-1;i_++) { rdy[n1+j-w1,i_] = rdy[n1+j-w1,i_] + v*rdx[i,i_]; } rdy[n1+j-w1,j] = rdy[n1+j-w1,j]+network.derror[i]; } continue; } if( network.structinfo[offs+0]<0 ) { bflag = false; if( (network.structinfo[offs+0]==-2 || network.structinfo[offs+0]==-3) || network.structinfo[offs+0]==-4 ) { // // Special neuron type, no back-propagation required // bflag = true; } alglib.ap.assert(bflag, "MLPHessianNBatch: unknown neuron type!"); continue; } } } } /************************************************************************* Internal subroutine Network must be processed by MLPProcess on X *************************************************************************/ private static void mlpinternalcalculategradient(multilayerperceptron network, double[] neurons, double[] weights, ref double[] derror, ref double[] grad, bool naturalerrorfunc, alglib.xparams _params) { int i = 0; int n1 = 0; int n2 = 0; int w1 = 0; int w2 = 0; int ntotal = 0; int istart = 0; int nin = 0; int nout = 0; int offs = 0; double dedf = 0; double dfdnet = 0; double v = 0; double fown = 0; double deown = 0; double net = 0; double mx = 0; bool bflag = new bool(); int i_ = 0; int i1_ = 0; // // Read network geometry // nin = network.structinfo[1]; nout = network.structinfo[2]; ntotal = network.structinfo[3]; istart = network.structinfo[5]; // // Pre-processing of dError/dOut: // from dError/dOut(normalized) to dError/dOut(non-normalized) // alglib.ap.assert(network.structinfo[6]==0 || network.structinfo[6]==1, "MLPInternalCalculateGradient: unknown normalization type!"); if( network.structinfo[6]==1 ) { // // Softmax // if( !naturalerrorfunc ) { mx = network.neurons[ntotal-nout]; for(i=0; i<=nout-1; i++) { mx = Math.Max(mx, network.neurons[ntotal-nout+i]); } net = 0; for(i=0; i<=nout-1; i++) { network.nwbuf[i] = Math.Exp(network.neurons[ntotal-nout+i]-mx); net = net+network.nwbuf[i]; } i1_ = (0)-(ntotal-nout); v = 0.0; for(i_=ntotal-nout; i_<=ntotal-1;i_++) { v += network.derror[i_]*network.nwbuf[i_+i1_]; } for(i=0; i<=nout-1; i++) { fown = network.nwbuf[i]; deown = network.derror[ntotal-nout+i]; network.nwbuf[nout+i] = (-v+deown*fown+deown*(net-fown))*fown/math.sqr(net); } for(i=0; i<=nout-1; i++) { network.derror[ntotal-nout+i] = network.nwbuf[nout+i]; } } } else { // // Un-standardisation // for(i=0; i<=nout-1; i++) { network.derror[ntotal-nout+i] = network.derror[ntotal-nout+i]*network.columnsigmas[nin+i]; } } // // Backpropagation // for(i=ntotal-1; i>=0; i--) { // // Extract info // offs = istart+i*nfieldwidth; if( network.structinfo[offs+0]>0 || network.structinfo[offs+0]==-5 ) { // // Activation function // dedf = network.derror[i]; dfdnet = network.dfdnet[i]; derror[network.structinfo[offs+2]] = derror[network.structinfo[offs+2]]+dedf*dfdnet; continue; } if( network.structinfo[offs+0]==0 ) { // // Adaptive summator // n1 = network.structinfo[offs+2]; n2 = n1+network.structinfo[offs+1]-1; w1 = network.structinfo[offs+3]; w2 = w1+network.structinfo[offs+1]-1; dedf = network.derror[i]; dfdnet = 1.0; v = dedf*dfdnet; i1_ = (n1) - (w1); for(i_=w1; i_<=w2;i_++) { grad[i_] = v*neurons[i_+i1_]; } i1_ = (w1) - (n1); for(i_=n1; i_<=n2;i_++) { derror[i_] = derror[i_] + v*weights[i_+i1_]; } continue; } if( network.structinfo[offs+0]<0 ) { bflag = false; if( (network.structinfo[offs+0]==-2 || network.structinfo[offs+0]==-3) || network.structinfo[offs+0]==-4 ) { // // Special neuron type, no back-propagation required // bflag = true; } alglib.ap.assert(bflag, "MLPInternalCalculateGradient: unknown neuron type!"); continue; } } } private static void mlpchunkedgradient(multilayerperceptron network, double[,] xy, int cstart, int csize, double[] batch4buf, double[] hpcbuf, ref double e, bool naturalerrorfunc, alglib.xparams _params) { int i = 0; int j = 0; int k = 0; int kl = 0; int ntotal = 0; int nin = 0; int nout = 0; int offs = 0; double f = 0; double df = 0; double d2f = 0; double v = 0; double vv = 0; double s = 0; double fown = 0; double deown = 0; bool bflag = new bool(); int istart = 0; int entrysize = 0; int dfoffs = 0; int derroroffs = 0; int entryoffs = 0; int neuronidx = 0; int srcentryoffs = 0; int srcneuronidx = 0; int srcweightidx = 0; int neurontype = 0; int nweights = 0; int offs0 = 0; int offs1 = 0; int offs2 = 0; double v0 = 0; double v1 = 0; double v2 = 0; double v3 = 0; double s0 = 0; double s1 = 0; double s2 = 0; double s3 = 0; int chunksize = 0; chunksize = 4; alglib.ap.assert(csize<=chunksize, "MLPChunkedGradient: internal error (CSize>ChunkSize)"); // // Try to use HPC core, if possible // if( hpccores.hpcchunkedgradient(network.weights, network.structinfo, network.columnmeans, network.columnsigmas, xy, cstart, csize, batch4buf, hpcbuf, ref e, naturalerrorfunc, _params) ) { return; } // // Read network geometry, prepare data // nin = network.structinfo[1]; nout = network.structinfo[2]; ntotal = network.structinfo[3]; istart = network.structinfo[5]; entrysize = 12; dfoffs = 4; derroroffs = 8; // // Fill Batch4Buf by zeros. // // THIS STAGE IS VERY IMPORTANT! // // We fill all components of entry - neuron values, dF/dNET, dError/dF. // It allows us to easily handle situations when CSize0 || neurontype==-5 ) { // // "activation function" neuron, which takes value of neuron SrcNeuronIdx // and applies activation function to it. // // This neuron has no weights and no tunable parameters. // srcneuronidx = network.structinfo[offs+2]; srcentryoffs = entrysize*srcneuronidx; mlpactivationfunction(batch4buf[srcentryoffs+0], neurontype, ref f, ref df, ref d2f, _params); batch4buf[entryoffs+0] = f; batch4buf[entryoffs+0+dfoffs] = df; mlpactivationfunction(batch4buf[srcentryoffs+1], neurontype, ref f, ref df, ref d2f, _params); batch4buf[entryoffs+1] = f; batch4buf[entryoffs+1+dfoffs] = df; mlpactivationfunction(batch4buf[srcentryoffs+2], neurontype, ref f, ref df, ref d2f, _params); batch4buf[entryoffs+2] = f; batch4buf[entryoffs+2+dfoffs] = df; mlpactivationfunction(batch4buf[srcentryoffs+3], neurontype, ref f, ref df, ref d2f, _params); batch4buf[entryoffs+3] = f; batch4buf[entryoffs+3+dfoffs] = df; continue; } if( neurontype==0 ) { // // "adaptive summator" neuron, whose output is a weighted sum of inputs. // It has weights, but has no activation function. // nweights = network.structinfo[offs+1]; srcneuronidx = network.structinfo[offs+2]; srcentryoffs = entrysize*srcneuronidx; srcweightidx = network.structinfo[offs+3]; v0 = 0; v1 = 0; v2 = 0; v3 = 0; for(j=0; j<=nweights-1; j++) { v = network.weights[srcweightidx]; srcweightidx = srcweightidx+1; v0 = v0+v*batch4buf[srcentryoffs+0]; v1 = v1+v*batch4buf[srcentryoffs+1]; v2 = v2+v*batch4buf[srcentryoffs+2]; v3 = v3+v*batch4buf[srcentryoffs+3]; srcentryoffs = srcentryoffs+entrysize; } batch4buf[entryoffs+0] = v0; batch4buf[entryoffs+1] = v1; batch4buf[entryoffs+2] = v2; batch4buf[entryoffs+3] = v3; batch4buf[entryoffs+0+dfoffs] = 1; batch4buf[entryoffs+1+dfoffs] = 1; batch4buf[entryoffs+2+dfoffs] = 1; batch4buf[entryoffs+3+dfoffs] = 1; continue; } if( neurontype<0 ) { bflag = false; if( neurontype==-2 ) { // // Input neuron, left unchanged // bflag = true; } if( neurontype==-3 ) { // // "-1" neuron // batch4buf[entryoffs+0] = -1; batch4buf[entryoffs+1] = -1; batch4buf[entryoffs+2] = -1; batch4buf[entryoffs+3] = -1; batch4buf[entryoffs+0+dfoffs] = 0; batch4buf[entryoffs+1+dfoffs] = 0; batch4buf[entryoffs+2+dfoffs] = 0; batch4buf[entryoffs+3+dfoffs] = 0; bflag = true; } if( neurontype==-4 ) { // // "0" neuron // batch4buf[entryoffs+0] = 0; batch4buf[entryoffs+1] = 0; batch4buf[entryoffs+2] = 0; batch4buf[entryoffs+3] = 0; batch4buf[entryoffs+0+dfoffs] = 0; batch4buf[entryoffs+1+dfoffs] = 0; batch4buf[entryoffs+2+dfoffs] = 0; batch4buf[entryoffs+3+dfoffs] = 0; bflag = true; } alglib.ap.assert(bflag, "MLPChunkedGradient: internal error - unknown neuron type!"); continue; } } // // Intermediate phase between forward and backward passes. // // For regression networks: // * forward pass is completely done (no additional post-processing is // needed). // * before starting backward pass, we have to calculate dError/dOut // for output neurons. We also update error at this phase. // // For classification networks: // * in addition to forward pass we apply SOFTMAX normalization to // output neurons. // * after applying normalization, we have to calculate dError/dOut, // which is calculated in two steps: // * first, we calculate derivative of error with respect to SOFTMAX // normalized outputs (normalized dError) // * then, we calculate derivative of error with respect to values // of outputs BEFORE normalization was applied to them // alglib.ap.assert(network.structinfo[6]==0 || network.structinfo[6]==1, "MLPChunkedGradient: unknown normalization type!"); if( network.structinfo[6]==1 ) { // // SOFTMAX-normalized network. // // First, calculate (V0,V1,V2,V3) - component-wise maximum // of output neurons. This vector of maximum values will be // used for normalization of outputs prior to calculating // exponentials. // // NOTE: the only purpose of this stage is to prevent overflow // during calculation of exponentials. With this stage // we make sure that all exponentials are calculated // with non-positive argument. If you load (0,0,0,0) to // (V0,V1,V2,V3), your program will continue working - // although with less robustness. // entryoffs = entrysize*(ntotal-nout); v0 = batch4buf[entryoffs+0]; v1 = batch4buf[entryoffs+1]; v2 = batch4buf[entryoffs+2]; v3 = batch4buf[entryoffs+3]; entryoffs = entryoffs+entrysize; for(i=1; i<=nout-1; i++) { v = batch4buf[entryoffs+0]; if( v>v0 ) { v0 = v; } v = batch4buf[entryoffs+1]; if( v>v1 ) { v1 = v; } v = batch4buf[entryoffs+2]; if( v>v2 ) { v2 = v; } v = batch4buf[entryoffs+3]; if( v>v3 ) { v3 = v; } entryoffs = entryoffs+entrysize; } // // Then, calculate exponentials and place them to part of the // array which is located past the last entry. We also // calculate sum of exponentials which will be stored past the // exponentials. // entryoffs = entrysize*(ntotal-nout); offs0 = entrysize*ntotal; s0 = 0; s1 = 0; s2 = 0; s3 = 0; for(i=0; i<=nout-1; i++) { v = Math.Exp(batch4buf[entryoffs+0]-v0); s0 = s0+v; batch4buf[offs0+0] = v; v = Math.Exp(batch4buf[entryoffs+1]-v1); s1 = s1+v; batch4buf[offs0+1] = v; v = Math.Exp(batch4buf[entryoffs+2]-v2); s2 = s2+v; batch4buf[offs0+2] = v; v = Math.Exp(batch4buf[entryoffs+3]-v3); s3 = s3+v; batch4buf[offs0+3] = v; entryoffs = entryoffs+entrysize; offs0 = offs0+chunksize; } offs0 = entrysize*ntotal+2*nout*chunksize; batch4buf[offs0+0] = s0; batch4buf[offs0+1] = s1; batch4buf[offs0+2] = s2; batch4buf[offs0+3] = s3; // // Now we have: // * Batch4Buf[0...EntrySize*NTotal-1] stores: // * NTotal*ChunkSize neuron output values (SOFTMAX normalization // was not applied to these values), // * NTotal*ChunkSize values of dF/dNET (derivative of neuron // output with respect to its input) // * NTotal*ChunkSize zeros in the elements which correspond to // dError/dOut (derivative of error with respect to neuron output). // * Batch4Buf[EntrySize*NTotal...EntrySize*NTotal+ChunkSize*NOut-1] - // stores exponentials of last NOut neurons. // * Batch4Buf[EntrySize*NTotal+ChunkSize*NOut-1...EntrySize*NTotal+ChunkSize*2*NOut-1] // - can be used for temporary calculations // * Batch4Buf[EntrySize*NTotal+ChunkSize*2*NOut...EntrySize*NTotal+ChunkSize*2*NOut+ChunkSize-1] // - stores sum-of-exponentials // // Block below calculates derivatives of error function with respect // to non-SOFTMAX-normalized output values of last NOut neurons. // // It is quite complicated; we do not describe algebra behind it, // but if you want you may check it yourself :) // if( naturalerrorfunc ) { // // Calculate derivative of error with respect to values of // output neurons PRIOR TO SOFTMAX NORMALIZATION. Because we // use natural error function (cross-entropy), we can do so // very easy. // offs0 = entrysize*ntotal+2*nout*chunksize; for(k=0; k<=csize-1; k++) { s = batch4buf[offs0+k]; kl = (int)Math.Round(xy[cstart+k,nin]); offs1 = (ntotal-nout)*entrysize+derroroffs+k; offs2 = entrysize*ntotal+k; for(i=0; i<=nout-1; i++) { if( i==kl ) { v = 1; } else { v = 0; } vv = batch4buf[offs2]; batch4buf[offs1] = vv/s-v; e = e+safecrossentropy(v, vv/s, _params); offs1 = offs1+entrysize; offs2 = offs2+chunksize; } } } else { // // SOFTMAX normalization makes things very difficult. // Sorry, we do not dare to describe this esoteric math // in details. // offs0 = entrysize*ntotal+chunksize*2*nout; for(k=0; k<=csize-1; k++) { s = batch4buf[offs0+k]; kl = (int)Math.Round(xy[cstart+k,nin]); vv = 0; offs1 = entrysize*ntotal+k; offs2 = entrysize*ntotal+nout*chunksize+k; for(i=0; i<=nout-1; i++) { fown = batch4buf[offs1]; if( i==kl ) { deown = fown/s-1; } else { deown = fown/s; } batch4buf[offs2] = deown; vv = vv+deown*fown; e = e+deown*deown/2; offs1 = offs1+chunksize; offs2 = offs2+chunksize; } offs1 = entrysize*ntotal+k; offs2 = entrysize*ntotal+nout*chunksize+k; for(i=0; i<=nout-1; i++) { fown = batch4buf[offs1]; deown = batch4buf[offs2]; batch4buf[(ntotal-nout+i)*entrysize+derroroffs+k] = (-vv+deown*fown+deown*(s-fown))*fown/math.sqr(s); offs1 = offs1+chunksize; offs2 = offs2+chunksize; } } } } else { // // Regression network with sum-of-squares function. // // For each NOut of last neurons: // * calculate difference between actual and desired output // * calculate dError/dOut for this neuron (proportional to difference) // * store in in last 4 components of entry (these values are used // to start backpropagation) // * update error // for(i=0; i<=nout-1; i++) { v0 = network.columnsigmas[nin+i]; v1 = network.columnmeans[nin+i]; entryoffs = entrysize*(ntotal-nout+i); offs0 = entryoffs; offs1 = entryoffs+derroroffs; for(j=0; j<=csize-1; j++) { v = batch4buf[offs0+j]*v0+v1-xy[cstart+j,nin+i]; batch4buf[offs1+j] = v*v0; e = e+v*v/2; } } } // // Backpropagation // for(neuronidx=ntotal-1; neuronidx>=0; neuronidx--) { entryoffs = entrysize*neuronidx; offs = istart+neuronidx*nfieldwidth; neurontype = network.structinfo[offs+0]; if( neurontype>0 || neurontype==-5 ) { // // Activation function // srcneuronidx = network.structinfo[offs+2]; srcentryoffs = entrysize*srcneuronidx; offs0 = srcentryoffs+derroroffs; offs1 = entryoffs+derroroffs; offs2 = entryoffs+dfoffs; batch4buf[offs0+0] = batch4buf[offs0+0]+batch4buf[offs1+0]*batch4buf[offs2+0]; batch4buf[offs0+1] = batch4buf[offs0+1]+batch4buf[offs1+1]*batch4buf[offs2+1]; batch4buf[offs0+2] = batch4buf[offs0+2]+batch4buf[offs1+2]*batch4buf[offs2+2]; batch4buf[offs0+3] = batch4buf[offs0+3]+batch4buf[offs1+3]*batch4buf[offs2+3]; continue; } if( neurontype==0 ) { // // Adaptive summator // nweights = network.structinfo[offs+1]; srcneuronidx = network.structinfo[offs+2]; srcentryoffs = entrysize*srcneuronidx; srcweightidx = network.structinfo[offs+3]; v0 = batch4buf[entryoffs+derroroffs+0]; v1 = batch4buf[entryoffs+derroroffs+1]; v2 = batch4buf[entryoffs+derroroffs+2]; v3 = batch4buf[entryoffs+derroroffs+3]; for(j=0; j<=nweights-1; j++) { offs0 = srcentryoffs; offs1 = srcentryoffs+derroroffs; v = network.weights[srcweightidx]; hpcbuf[srcweightidx] = hpcbuf[srcweightidx]+batch4buf[offs0+0]*v0+batch4buf[offs0+1]*v1+batch4buf[offs0+2]*v2+batch4buf[offs0+3]*v3; batch4buf[offs1+0] = batch4buf[offs1+0]+v*v0; batch4buf[offs1+1] = batch4buf[offs1+1]+v*v1; batch4buf[offs1+2] = batch4buf[offs1+2]+v*v2; batch4buf[offs1+3] = batch4buf[offs1+3]+v*v3; srcentryoffs = srcentryoffs+entrysize; srcweightidx = srcweightidx+1; } continue; } if( neurontype<0 ) { bflag = false; if( (neurontype==-2 || neurontype==-3) || neurontype==-4 ) { // // Special neuron type, no back-propagation required // bflag = true; } alglib.ap.assert(bflag, "MLPInternalCalculateGradient: unknown neuron type!"); continue; } } } private static void mlpchunkedprocess(multilayerperceptron network, double[,] xy, int cstart, int csize, double[] batch4buf, double[] hpcbuf, alglib.xparams _params) { int i = 0; int j = 0; int ntotal = 0; int nin = 0; int nout = 0; int offs = 0; double f = 0; double df = 0; double d2f = 0; double v = 0; bool bflag = new bool(); int istart = 0; int entrysize = 0; int entryoffs = 0; int neuronidx = 0; int srcentryoffs = 0; int srcneuronidx = 0; int srcweightidx = 0; int neurontype = 0; int nweights = 0; int offs0 = 0; double v0 = 0; double v1 = 0; double v2 = 0; double v3 = 0; double s0 = 0; double s1 = 0; double s2 = 0; double s3 = 0; int chunksize = 0; chunksize = 4; alglib.ap.assert(csize<=chunksize, "MLPChunkedProcess: internal error (CSize>ChunkSize)"); // // Try to use HPC core, if possible // if( hpccores.hpcchunkedprocess(network.weights, network.structinfo, network.columnmeans, network.columnsigmas, xy, cstart, csize, batch4buf, hpcbuf, _params) ) { return; } // // Read network geometry, prepare data // nin = network.structinfo[1]; nout = network.structinfo[2]; ntotal = network.structinfo[3]; istart = network.structinfo[5]; entrysize = 4; // // Fill Batch4Buf by zeros. // // THIS STAGE IS VERY IMPORTANT! // // We fill all components of entry - neuron values, dF/dNET, dError/dF. // It allows us to easily handle situations when CSize0 || neurontype==-5 ) { // // "activation function" neuron, which takes value of neuron SrcNeuronIdx // and applies activation function to it. // // This neuron has no weights and no tunable parameters. // srcneuronidx = network.structinfo[offs+2]; srcentryoffs = entrysize*srcneuronidx; mlpactivationfunction(batch4buf[srcentryoffs+0], neurontype, ref f, ref df, ref d2f, _params); batch4buf[entryoffs+0] = f; mlpactivationfunction(batch4buf[srcentryoffs+1], neurontype, ref f, ref df, ref d2f, _params); batch4buf[entryoffs+1] = f; mlpactivationfunction(batch4buf[srcentryoffs+2], neurontype, ref f, ref df, ref d2f, _params); batch4buf[entryoffs+2] = f; mlpactivationfunction(batch4buf[srcentryoffs+3], neurontype, ref f, ref df, ref d2f, _params); batch4buf[entryoffs+3] = f; continue; } if( neurontype==0 ) { // // "adaptive summator" neuron, whose output is a weighted sum of inputs. // It has weights, but has no activation function. // nweights = network.structinfo[offs+1]; srcneuronidx = network.structinfo[offs+2]; srcentryoffs = entrysize*srcneuronidx; srcweightidx = network.structinfo[offs+3]; v0 = 0; v1 = 0; v2 = 0; v3 = 0; for(j=0; j<=nweights-1; j++) { v = network.weights[srcweightidx]; srcweightidx = srcweightidx+1; v0 = v0+v*batch4buf[srcentryoffs+0]; v1 = v1+v*batch4buf[srcentryoffs+1]; v2 = v2+v*batch4buf[srcentryoffs+2]; v3 = v3+v*batch4buf[srcentryoffs+3]; srcentryoffs = srcentryoffs+entrysize; } batch4buf[entryoffs+0] = v0; batch4buf[entryoffs+1] = v1; batch4buf[entryoffs+2] = v2; batch4buf[entryoffs+3] = v3; continue; } if( neurontype<0 ) { bflag = false; if( neurontype==-2 ) { // // Input neuron, left unchanged // bflag = true; } if( neurontype==-3 ) { // // "-1" neuron // batch4buf[entryoffs+0] = -1; batch4buf[entryoffs+1] = -1; batch4buf[entryoffs+2] = -1; batch4buf[entryoffs+3] = -1; bflag = true; } if( neurontype==-4 ) { // // "0" neuron // batch4buf[entryoffs+0] = 0; batch4buf[entryoffs+1] = 0; batch4buf[entryoffs+2] = 0; batch4buf[entryoffs+3] = 0; bflag = true; } alglib.ap.assert(bflag, "MLPChunkedProcess: internal error - unknown neuron type!"); continue; } } // // SOFTMAX normalization or scaling. // alglib.ap.assert(network.structinfo[6]==0 || network.structinfo[6]==1, "MLPChunkedProcess: unknown normalization type!"); if( network.structinfo[6]==1 ) { // // SOFTMAX-normalized network. // // First, calculate (V0,V1,V2,V3) - component-wise maximum // of output neurons. This vector of maximum values will be // used for normalization of outputs prior to calculating // exponentials. // // NOTE: the only purpose of this stage is to prevent overflow // during calculation of exponentials. With this stage // we make sure that all exponentials are calculated // with non-positive argument. If you load (0,0,0,0) to // (V0,V1,V2,V3), your program will continue working - // although with less robustness. // entryoffs = entrysize*(ntotal-nout); v0 = batch4buf[entryoffs+0]; v1 = batch4buf[entryoffs+1]; v2 = batch4buf[entryoffs+2]; v3 = batch4buf[entryoffs+3]; entryoffs = entryoffs+entrysize; for(i=1; i<=nout-1; i++) { v = batch4buf[entryoffs+0]; if( v>v0 ) { v0 = v; } v = batch4buf[entryoffs+1]; if( v>v1 ) { v1 = v; } v = batch4buf[entryoffs+2]; if( v>v2 ) { v2 = v; } v = batch4buf[entryoffs+3]; if( v>v3 ) { v3 = v; } entryoffs = entryoffs+entrysize; } // // Then, calculate exponentials and place them to part of the // array which is located past the last entry. We also // calculate sum of exponentials. // entryoffs = entrysize*(ntotal-nout); offs0 = entrysize*ntotal; s0 = 0; s1 = 0; s2 = 0; s3 = 0; for(i=0; i<=nout-1; i++) { v = Math.Exp(batch4buf[entryoffs+0]-v0); s0 = s0+v; batch4buf[offs0+0] = v; v = Math.Exp(batch4buf[entryoffs+1]-v1); s1 = s1+v; batch4buf[offs0+1] = v; v = Math.Exp(batch4buf[entryoffs+2]-v2); s2 = s2+v; batch4buf[offs0+2] = v; v = Math.Exp(batch4buf[entryoffs+3]-v3); s3 = s3+v; batch4buf[offs0+3] = v; entryoffs = entryoffs+entrysize; offs0 = offs0+chunksize; } // // Write SOFTMAX-normalized values to the output array. // offs0 = entrysize*ntotal; for(i=0; i<=nout-1; i++) { if( csize>0 ) { xy[cstart+0,nin+i] = batch4buf[offs0+0]/s0; } if( csize>1 ) { xy[cstart+1,nin+i] = batch4buf[offs0+1]/s1; } if( csize>2 ) { xy[cstart+2,nin+i] = batch4buf[offs0+2]/s2; } if( csize>3 ) { xy[cstart+3,nin+i] = batch4buf[offs0+3]/s3; } offs0 = offs0+chunksize; } } else { // // Regression network with sum-of-squares function. // // For each NOut of last neurons: // * calculate difference between actual and desired output // * calculate dError/dOut for this neuron (proportional to difference) // * store in in last 4 components of entry (these values are used // to start backpropagation) // * update error // for(i=0; i<=nout-1; i++) { v0 = network.columnsigmas[nin+i]; v1 = network.columnmeans[nin+i]; entryoffs = entrysize*(ntotal-nout+i); for(j=0; j<=csize-1; j++) { xy[cstart+j,nin+i] = batch4buf[entryoffs+j]*v0+v1; } } } } /************************************************************************* Returns T*Ln(T/Z), guarded against overflow/underflow. Internal subroutine. *************************************************************************/ private static double safecrossentropy(double t, double z, alglib.xparams _params) { double result = 0; double r = 0; if( (double)(t)==(double)(0) ) { result = 0; } else { if( (double)(Math.Abs(z))>(double)(1) ) { // // Shouldn't be the case with softmax, // but we just want to be sure. // if( (double)(t/z)==(double)(0) ) { r = math.minrealnumber; } else { r = t/z; } } else { // // Normal case // if( (double)(z)==(double)(0) || (double)(Math.Abs(t))>=(double)(math.maxrealnumber*Math.Abs(z)) ) { r = math.maxrealnumber; } else { r = t/z; } } result = t*Math.Log(r); } return result; } /************************************************************************* This function performs backward pass of neural network randimization: * it assumes that Network.Weights stores standard deviation of weights (weights are not generated yet, only their deviations are present) * it sets deviations of weights which feed NeuronIdx-th neuron to specified value * it recursively passes to deeper neuron and modifies their weights * it stops after encountering nonlinear neurons, linear activation function, input neurons, "0" and "-1" neurons -- ALGLIB -- Copyright 27.06.2013 by Bochkanov Sergey *************************************************************************/ private static void randomizebackwardpass(multilayerperceptron network, int neuronidx, double v, alglib.xparams _params) { int istart = 0; int neurontype = 0; int n1 = 0; int n2 = 0; int w1 = 0; int w2 = 0; int offs = 0; int i = 0; istart = network.structinfo[5]; neurontype = network.structinfo[istart+neuronidx*nfieldwidth+0]; if( neurontype==-2 ) { // // Input neuron - stop // return; } if( neurontype==-3 ) { // // "-1" neuron: stop // return; } if( neurontype==-4 ) { // // "0" neuron: stop // return; } if( neurontype==0 ) { // // Adaptive summator neuron: // * modify deviations of its weights // * recursively call this function for its inputs // offs = istart+neuronidx*nfieldwidth; n1 = network.structinfo[offs+2]; n2 = n1+network.structinfo[offs+1]-1; w1 = network.structinfo[offs+3]; w2 = w1+network.structinfo[offs+1]-1; for(i=w1; i<=w2; i++) { network.weights[i] = v; } for(i=n1; i<=n2; i++) { randomizebackwardpass(network, i, v, _params); } return; } if( neurontype==-5 ) { // // Linear activation function: stop // return; } if( neurontype>0 ) { // // Nonlinear activation function: stop // return; } alglib.ap.assert(false, "RandomizeBackwardPass: unexpected neuron type"); } } public class lda { /************************************************************************* Multiclass Fisher LDA Subroutine finds coefficients of linear combination which optimally separates training set on classes. COMMERCIAL EDITION OF ALGLIB: ! Commercial version of ALGLIB includes two important improvements of ! this function, which can be used from C++ and C#: ! * Intel MKL support (lightweight Intel MKL is shipped with ALGLIB) ! * multithreading support ! ! Intel MKL gives approximately constant (with respect to number of ! worker threads) acceleration factor which depends on CPU being used, ! problem size and "baseline" ALGLIB edition which is used for ! comparison. Best results are achieved for high-dimensional problems ! (NVars is at least 256). ! ! Multithreading is used to accelerate initial phase of LDA, which ! includes calculation of products of large matrices. Again, for best ! efficiency problem must be high-dimensional. ! ! Generally, commercial ALGLIB is several times faster than open-source ! generic C edition, and many times faster than open-source C# edition. ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: XY - training set, array[0..NPoints-1,0..NVars]. First NVars columns store values of independent variables, next column stores number of class (from 0 to NClasses-1) which dataset element belongs to. Fractional values are rounded to nearest integer. NPoints - training set size, NPoints>=0 NVars - number of independent variables, NVars>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: Info - return code: * -4, if internal EVD subroutine hasn't converged * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, NVars<1, NClasses<2) * 1, if task has been solved * 2, if there was a multicollinearity in training set, but task has been solved. W - linear combination coefficients, array[0..NVars-1] -- ALGLIB -- Copyright 31.05.2008 by Bochkanov Sergey *************************************************************************/ public static void fisherlda(double[,] xy, int npoints, int nvars, int nclasses, ref int info, ref double[] w, alglib.xparams _params) { double[,] w2 = new double[0,0]; int i_ = 0; info = 0; w = new double[0]; fisherldan(xy, npoints, nvars, nclasses, ref info, ref w2, _params); if( info>0 ) { w = new double[nvars]; for(i_=0; i_<=nvars-1;i_++) { w[i_] = w2[i_,0]; } } } /************************************************************************* N-dimensional multiclass Fisher LDA Subroutine finds coefficients of linear combinations which optimally separates training set on classes. It returns N-dimensional basis whose vector are sorted by quality of training set separation (in descending order). ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: XY - training set, array[0..NPoints-1,0..NVars]. First NVars columns store values of independent variables, next column stores number of class (from 0 to NClasses-1) which dataset element belongs to. Fractional values are rounded to nearest integer. NPoints - training set size, NPoints>=0 NVars - number of independent variables, NVars>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: Info - return code: * -4, if internal EVD subroutine hasn't converged * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, NVars<1, NClasses<2) * 1, if task has been solved * 2, if there was a multicollinearity in training set, but task has been solved. W - basis, array[0..NVars-1,0..NVars-1] columns of matrix stores basis vectors, sorted by quality of training set separation (in descending order) -- ALGLIB -- Copyright 31.05.2008 by Bochkanov Sergey *************************************************************************/ public static void fisherldan(double[,] xy, int npoints, int nvars, int nclasses, ref int info, ref double[,] w, alglib.xparams _params) { int i = 0; int j = 0; int k = 0; int m = 0; double v = 0; int[] c = new int[0]; double[] mu = new double[0]; double[,] muc = new double[0,0]; int[] nc = new int[0]; double[,] sw = new double[0,0]; double[,] st = new double[0,0]; double[,] z = new double[0,0]; double[,] z2 = new double[0,0]; double[,] tm = new double[0,0]; double[,] sbroot = new double[0,0]; double[,] a = new double[0,0]; double[,] xyc = new double[0,0]; double[,] xyproj = new double[0,0]; double[,] wproj = new double[0,0]; double[] tf = new double[0]; double[] d = new double[0]; double[] d2 = new double[0]; double[] work = new double[0]; int i_ = 0; info = 0; w = new double[0,0]; // // Test data // if( (npoints<0 || nvars<1) || nclasses<2 ) { info = -1; return; } for(i=0; i<=npoints-1; i++) { if( (int)Math.Round(xy[i,nvars])<0 || (int)Math.Round(xy[i,nvars])>=nclasses ) { info = -2; return; } } info = 1; // // Special case: NPoints<=1 // Degenerate task. // if( npoints<=1 ) { info = 2; w = new double[nvars, nvars]; for(i=0; i<=nvars-1; i++) { for(j=0; j<=nvars-1; j++) { if( i==j ) { w[i,j] = 1; } else { w[i,j] = 0; } } } return; } // // Prepare temporaries // tf = new double[nvars]; work = new double[Math.Max(nvars, npoints)+1]; xyc = new double[npoints, nvars]; // // Convert class labels from reals to integers (just for convenience) // c = new int[npoints]; for(i=0; i<=npoints-1; i++) { c[i] = (int)Math.Round(xy[i,nvars]); } // // Calculate class sizes, class means // mu = new double[nvars]; muc = new double[nclasses, nvars]; nc = new int[nclasses]; for(j=0; j<=nvars-1; j++) { mu[j] = 0; } for(i=0; i<=nclasses-1; i++) { nc[i] = 0; for(j=0; j<=nvars-1; j++) { muc[i,j] = 0; } } for(i=0; i<=npoints-1; i++) { for(i_=0; i_<=nvars-1;i_++) { mu[i_] = mu[i_] + xy[i,i_]; } for(i_=0; i_<=nvars-1;i_++) { muc[c[i],i_] = muc[c[i],i_] + xy[i,i_]; } nc[c[i]] = nc[c[i]]+1; } for(i=0; i<=nclasses-1; i++) { v = (double)1/(double)nc[i]; for(i_=0; i_<=nvars-1;i_++) { muc[i,i_] = v*muc[i,i_]; } } v = (double)1/(double)npoints; for(i_=0; i_<=nvars-1;i_++) { mu[i_] = v*mu[i_]; } // // Create ST matrix // st = new double[nvars, nvars]; for(i=0; i<=nvars-1; i++) { for(j=0; j<=nvars-1; j++) { st[i,j] = 0; } } for(k=0; k<=npoints-1; k++) { for(i_=0; i_<=nvars-1;i_++) { xyc[k,i_] = xy[k,i_]; } for(i_=0; i_<=nvars-1;i_++) { xyc[k,i_] = xyc[k,i_] - mu[i_]; } } ablas.rmatrixgemm(nvars, nvars, npoints, 1.0, xyc, 0, 0, 1, xyc, 0, 0, 0, 0.0, st, 0, 0, _params); // // Create SW matrix // sw = new double[nvars, nvars]; for(i=0; i<=nvars-1; i++) { for(j=0; j<=nvars-1; j++) { sw[i,j] = 0; } } for(k=0; k<=npoints-1; k++) { for(i_=0; i_<=nvars-1;i_++) { xyc[k,i_] = xy[k,i_]; } for(i_=0; i_<=nvars-1;i_++) { xyc[k,i_] = xyc[k,i_] - muc[c[k],i_]; } } ablas.rmatrixgemm(nvars, nvars, npoints, 1.0, xyc, 0, 0, 1, xyc, 0, 0, 0, 0.0, sw, 0, 0, _params); // // Maximize ratio J=(w'*ST*w)/(w'*SW*w). // // First, make transition from w to v such that w'*ST*w becomes v'*v: // v = root(ST)*w = R*w // R = root(D)*Z' // w = (root(ST)^-1)*v = RI*v // RI = Z*inv(root(D)) // J = (v'*v)/(v'*(RI'*SW*RI)*v) // ST = Z*D*Z' // // so we have // // J = (v'*v) / (v'*(inv(root(D))*Z'*SW*Z*inv(root(D)))*v) = // = (v'*v) / (v'*A*v) // if( !evd.smatrixevd(st, nvars, 1, true, ref d, ref z, _params) ) { info = -4; return; } w = new double[nvars, nvars]; if( (double)(d[nvars-1])<=(double)(0) || (double)(d[0])<=(double)(1000*math.machineepsilon*d[nvars-1]) ) { // // Special case: D[NVars-1]<=0 // Degenerate task (all variables takes the same value). // if( (double)(d[nvars-1])<=(double)(0) ) { info = 2; for(i=0; i<=nvars-1; i++) { for(j=0; j<=nvars-1; j++) { if( i==j ) { w[i,j] = 1; } else { w[i,j] = 0; } } } return; } // // Special case: degenerate ST matrix, multicollinearity found. // Since we know ST eigenvalues/vectors we can translate task to // non-degenerate form. // // Let WG is orthogonal basis of the non zero variance subspace // of the ST and let WZ is orthogonal basis of the zero variance // subspace. // // Projection on WG allows us to use LDA on reduced M-dimensional // subspace, N-M vectors of WZ allows us to update reduced LDA // factors to full N-dimensional subspace. // m = 0; for(k=0; k<=nvars-1; k++) { if( (double)(d[k])<=(double)(1000*math.machineepsilon*d[nvars-1]) ) { m = k+1; } } alglib.ap.assert(m!=0, "FisherLDAN: internal error #1"); xyproj = new double[npoints, nvars-m+1]; ablas.rmatrixgemm(npoints, nvars-m, nvars, 1.0, xy, 0, 0, 0, z, 0, m, 0, 0.0, xyproj, 0, 0, _params); for(i=0; i<=npoints-1; i++) { xyproj[i,nvars-m] = xy[i,nvars]; } fisherldan(xyproj, npoints, nvars-m, nclasses, ref info, ref wproj, _params); if( info<0 ) { return; } ablas.rmatrixgemm(nvars, nvars-m, nvars-m, 1.0, z, 0, m, 0, wproj, 0, 0, 0, 0.0, w, 0, 0, _params); for(k=nvars-m; k<=nvars-1; k++) { for(i_=0; i_<=nvars-1;i_++) { w[i_,k] = z[i_,k-(nvars-m)]; } } info = 2; } else { // // General case: no multicollinearity // tm = new double[nvars, nvars]; a = new double[nvars, nvars]; ablas.rmatrixgemm(nvars, nvars, nvars, 1.0, sw, 0, 0, 0, z, 0, 0, 0, 0.0, tm, 0, 0, _params); ablas.rmatrixgemm(nvars, nvars, nvars, 1.0, z, 0, 0, 1, tm, 0, 0, 0, 0.0, a, 0, 0, _params); for(i=0; i<=nvars-1; i++) { for(j=0; j<=nvars-1; j++) { a[i,j] = a[i,j]/Math.Sqrt(d[i]*d[j]); } } if( !evd.smatrixevd(a, nvars, 1, true, ref d2, ref z2, _params) ) { info = -4; return; } for(i=0; i<=nvars-1; i++) { for(k=0; k<=nvars-1; k++) { z2[i,k] = z2[i,k]/Math.Sqrt(d[i]); } } ablas.rmatrixgemm(nvars, nvars, nvars, 1.0, z, 0, 0, 0, z2, 0, 0, 0, 0.0, w, 0, 0, _params); } // // Post-processing: // * normalization // * converting to non-negative form, if possible // for(k=0; k<=nvars-1; k++) { v = 0.0; for(i_=0; i_<=nvars-1;i_++) { v += w[i_,k]*w[i_,k]; } v = 1/Math.Sqrt(v); for(i_=0; i_<=nvars-1;i_++) { w[i_,k] = v*w[i_,k]; } v = 0; for(i=0; i<=nvars-1; i++) { v = v+w[i,k]; } if( (double)(v)<(double)(0) ) { for(i_=0; i_<=nvars-1;i_++) { w[i_,k] = -1*w[i_,k]; } } } } } public class ssa { /************************************************************************* This object stores state of the SSA model. You should use ALGLIB functions to work with this object. *************************************************************************/ public class ssamodel : apobject { public int nsequences; public int[] sequenceidx; public double[] sequencedata; public int algotype; public int windowwidth; public int rtpowerup; public int topk; public int precomputedwidth; public int precomputednbasis; public double[,] precomputedbasis; public int defaultsubspaceits; public int memorylimit; public bool arebasisandsolvervalid; public double[,] basis; public double[,] basist; public double[] sv; public double[] forecasta; public int nbasis; public evd.eigsubspacestate solver; public double[,] xxt; public hqrnd.hqrndstate rs; public int rngseed; public int[] rtqueue; public int rtqueuecnt; public int rtqueuechunk; public int dbgcntevd; public double[] tmp0; public double[] tmp1; public evd.eigsubspacereport solverrep; public double[] alongtrend; public double[] alongnoise; public double[,] aseqtrajectory; public double[,] aseqtbproduct; public int[] aseqcounts; public double[] fctrend; public double[] fcnoise; public double[,] fctrendm; public double[,] uxbatch; public int uxbatchwidth; public int uxbatchsize; public int uxbatchlimit; public ssamodel() { init(); } public override void init() { sequenceidx = new int[0]; sequencedata = new double[0]; precomputedbasis = new double[0,0]; basis = new double[0,0]; basist = new double[0,0]; sv = new double[0]; forecasta = new double[0]; solver = new evd.eigsubspacestate(); xxt = new double[0,0]; rs = new hqrnd.hqrndstate(); rtqueue = new int[0]; tmp0 = new double[0]; tmp1 = new double[0]; solverrep = new evd.eigsubspacereport(); alongtrend = new double[0]; alongnoise = new double[0]; aseqtrajectory = new double[0,0]; aseqtbproduct = new double[0,0]; aseqcounts = new int[0]; fctrend = new double[0]; fcnoise = new double[0]; fctrendm = new double[0,0]; uxbatch = new double[0,0]; } public override alglib.apobject make_copy() { ssamodel _result = new ssamodel(); _result.nsequences = nsequences; _result.sequenceidx = (int[])sequenceidx.Clone(); _result.sequencedata = (double[])sequencedata.Clone(); _result.algotype = algotype; _result.windowwidth = windowwidth; _result.rtpowerup = rtpowerup; _result.topk = topk; _result.precomputedwidth = precomputedwidth; _result.precomputednbasis = precomputednbasis; _result.precomputedbasis = (double[,])precomputedbasis.Clone(); _result.defaultsubspaceits = defaultsubspaceits; _result.memorylimit = memorylimit; _result.arebasisandsolvervalid = arebasisandsolvervalid; _result.basis = (double[,])basis.Clone(); _result.basist = (double[,])basist.Clone(); _result.sv = (double[])sv.Clone(); _result.forecasta = (double[])forecasta.Clone(); _result.nbasis = nbasis; _result.solver = (evd.eigsubspacestate)solver.make_copy(); _result.xxt = (double[,])xxt.Clone(); _result.rs = (hqrnd.hqrndstate)rs.make_copy(); _result.rngseed = rngseed; _result.rtqueue = (int[])rtqueue.Clone(); _result.rtqueuecnt = rtqueuecnt; _result.rtqueuechunk = rtqueuechunk; _result.dbgcntevd = dbgcntevd; _result.tmp0 = (double[])tmp0.Clone(); _result.tmp1 = (double[])tmp1.Clone(); _result.solverrep = (evd.eigsubspacereport)solverrep.make_copy(); _result.alongtrend = (double[])alongtrend.Clone(); _result.alongnoise = (double[])alongnoise.Clone(); _result.aseqtrajectory = (double[,])aseqtrajectory.Clone(); _result.aseqtbproduct = (double[,])aseqtbproduct.Clone(); _result.aseqcounts = (int[])aseqcounts.Clone(); _result.fctrend = (double[])fctrend.Clone(); _result.fcnoise = (double[])fcnoise.Clone(); _result.fctrendm = (double[,])fctrendm.Clone(); _result.uxbatch = (double[,])uxbatch.Clone(); _result.uxbatchwidth = uxbatchwidth; _result.uxbatchsize = uxbatchsize; _result.uxbatchlimit = uxbatchlimit; return _result; } }; /************************************************************************* This function creates SSA model object. Right after creation model is in "dummy" mode - you can add data, but analyzing/prediction will return just zeros (it assumes that basis is empty). HOW TO USE SSA MODEL: 1. create model with ssacreate() 2. add data with one/many ssaaddsequence() calls 3. choose SSA algorithm with one of ssasetalgo...() functions: * ssasetalgotopkdirect() for direct one-run analysis * ssasetalgotopkrealtime() for algorithm optimized for many subsequent runs with warm-start capabilities * ssasetalgoprecomputed() for user-supplied basis 4. set window width with ssasetwindow() 5. perform one of the analysis-related activities: a) call ssagetbasis() to get basis b) call ssaanalyzelast() ssaanalyzesequence() or ssaanalyzelastwindow() to perform analysis (trend/noise separation) c) call one of the forecasting functions (ssaforecastlast() or ssaforecastsequence()) to perform prediction; alternatively, you can extract linear recurrence coefficients with ssagetlrr(). SSA analysis will be performed during first call to analysis-related function. SSA model is smart enough to track all changes in the dataset and model settings, to cache previously computed basis and to re-evaluate basis only when necessary. Additionally, if your setting involves constant stream of incoming data, you can perform quick update already calculated model with one of the incremental append-and-update functions: ssaappendpointandupdate() or ssaappendsequenceandupdate(). NOTE: steps (2), (3), (4) can be performed in arbitrary order. INPUT PARAMETERS: none OUTPUT PARAMETERS: S - structure which stores model state -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssacreate(ssamodel s, alglib.xparams _params) { // // Model data, algorithms and settings // s.nsequences = 0; s.sequenceidx = new int[1]; s.sequenceidx[0] = 0; s.algotype = 0; s.windowwidth = 1; s.rtpowerup = 1; s.arebasisandsolvervalid = false; s.rngseed = 1; s.defaultsubspaceits = 10; s.memorylimit = 50000000; // // Debug counters // s.dbgcntevd = 0; } /************************************************************************* This function sets window width for SSA model. You should call it before analysis phase. Default window width is 1 (not for real use). Special notes: * this function call can be performed at any moment before first call to analysis-related functions * changing window width invalidates internally stored basis; if you change window width AFTER you call analysis-related function, next analysis phase will require re-calculation of the basis according to current algorithm. * calling this function with exactly same window width as current one has no effect * if you specify window width larger than any data sequence stored in the model, analysis will return zero basis. INPUT PARAMETERS: S - SSA model created with ssacreate() WindowWidth - >=1, new window width OUTPUT PARAMETERS: S - SSA model, updated -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetwindow(ssamodel s, int windowwidth, alglib.xparams _params) { alglib.ap.assert(windowwidth>=1, "SSASetWindow: WindowWidth<1"); if( windowwidth==s.windowwidth ) { return; } s.windowwidth = windowwidth; s.arebasisandsolvervalid = false; } /************************************************************************* This function sets seed which is used to initialize internal RNG when we make pseudorandom decisions on model updates. By default, deterministic seed is used - which results in same sequence of pseudorandom decisions every time you run SSA model. If you specify non- deterministic seed value, then SSA model may return slightly different results after each run. This function can be useful when you have several SSA models updated with sseappendpointandupdate() called with 01 means that delayed power-up is performed -- ALGLIB -- Copyright 03.11.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetpoweruplength(ssamodel s, int pwlen, alglib.xparams _params) { alglib.ap.assert(pwlen>=0, "SSASetPowerUpLength: PWLen<0"); s.rtpowerup = Math.Max(pwlen, 1); s.arebasisandsolvervalid = false; } /************************************************************************* This function sets memory limit of SSA analysis. Straightforward SSA with sequence length T and window width W needs O(T*W) memory. It is possible to reduce memory consumption by splitting task into smaller chunks. Thus function allows you to specify approximate memory limit (measured in double precision numbers used for buffers). Actual memory consumption will be comparable to the number specified by you. Default memory limit is 50.000.000 (400Mbytes) in current version. INPUT PARAMETERS: S - SSA model MemLimit- memory limit, >=0. Zero value means no limit. -- ALGLIB -- Copyright 20.12.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetmemorylimit(ssamodel s, int memlimit, alglib.xparams _params) { if( memlimit<0 ) { memlimit = 0; } s.memorylimit = memlimit; } /************************************************************************* This function adds data sequence to SSA model. Only single-dimensional sequences are supported. What is a sequences? Following definitions/requirements apply: * a sequence is an array of values measured in subsequent, equally separated time moments (ticks). * you may have many sequences in your dataset; say, one sequence may correspond to one trading session. * sequence length should be larger than current window length (shorter sequences will be ignored during analysis). * analysis is performed within a sequence; different sequences are NOT stacked together to produce one large contiguous stream of data. * analysis is performed for all sequences at once, i.e. same set of basis vectors is computed for all sequences INCREMENTAL ANALYSIS This function is non intended for incremental updates of previously found SSA basis. Calling it invalidates all previous analysis results (basis is reset and will be recalculated from zero during next analysis). If you want to perform incremental/real-time SSA, consider using following functions: * ssaappendpointandupdate() for appending one point * ssaappendsequenceandupdate() for appending new sequence INPUT PARAMETERS: S - SSA model created with ssacreate() X - array[N], data, can be larger (additional values are ignored) N - data length, can be automatically determined from the array length. N>=0. OUTPUT PARAMETERS: S - SSA model, updated NOTE: you can clear dataset with ssacleardata() -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaaddsequence(ssamodel s, double[] x, int n, alglib.xparams _params) { int i = 0; int offs = 0; alglib.ap.assert(n>=0, "SSAAddSequence: N<0"); alglib.ap.assert(alglib.ap.len(x)>=n, "SSAAddSequence: X is too short"); alglib.ap.assert(apserv.isfinitevector(x, n, _params), "SSAAddSequence: X contains infinities NANs"); // // Invalidate model // s.arebasisandsolvervalid = false; // // Add sequence // apserv.ivectorgrowto(ref s.sequenceidx, s.nsequences+2, _params); s.sequenceidx[s.nsequences+1] = s.sequenceidx[s.nsequences]+n; apserv.rvectorgrowto(ref s.sequencedata, s.sequenceidx[s.nsequences+1], _params); offs = s.sequenceidx[s.nsequences]; for(i=0; i<=n-1; i++) { s.sequencedata[offs+i] = x[i]; } apserv.inc(ref s.nsequences, _params); } /************************************************************************* This function appends single point to last data sequence stored in the SSA model and tries to update model in the incremental manner (if possible with current algorithm). If you want to add more than one point at once: * if you want to add M points to the same sequence, perform M-1 calls with UpdateIts parameter set to 0.0, and last call with non-zero UpdateIts. * if you want to add new sequence, use ssaappendsequenceandupdate() Running time of this function does NOT depend on dataset size, only on window width and number of singular vectors. Depending on algorithm being used, incremental update has complexity: * for top-K real time - O(UpdateIts*K*Width^2), with fractional UpdateIts * for top-K direct - O(Width^3) for any non-zero UpdateIts * for precomputed basis - O(1), no update is performed INPUT PARAMETERS: S - SSA model created with ssacreate() X - new point UpdateIts - >=0, floating point (!) value, desired update frequency: * zero value means that point is stored, but no update is performed * integer part of the value means that specified number of iterations is always performed * fractional part of the value means that one iteration is performed with this probability. Recommended value: 0=(double)(0), "SSAAppendPointAndUpdate: UpdateIts<0"); alglib.ap.assert(s.nsequences>0, "SSAAppendPointAndUpdate: dataset is empty, no sequence to modify"); // // Append point to dataset // apserv.rvectorgrowto(ref s.sequencedata, s.sequenceidx[s.nsequences]+1, _params); s.sequencedata[s.sequenceidx[s.nsequences]] = x; s.sequenceidx[s.nsequences] = s.sequenceidx[s.nsequences]+1; // // Do we have something to analyze? If no, invalidate basis // (just to be sure) and exit. // if( !hassomethingtoanalyze(s, _params) ) { s.arebasisandsolvervalid = false; return; } // // Well, we have data to analyze and algorithm set, but basis is // invalid. Let's calculate it from scratch and exit. // if( !s.arebasisandsolvervalid ) { updatebasis(s, 0, 0.0, _params); return; } // // Update already computed basis // updatebasis(s, 1, updateits, _params); } /************************************************************************* This function appends new sequence to dataset stored in the SSA model and tries to update model in the incremental manner (if possible with current algorithm). Notes: * if you want to add M sequences at once, perform M-1 calls with UpdateIts parameter set to 0.0, and last call with non-zero UpdateIts. * if you want to add just one point, use ssaappendpointandupdate() Running time of this function does NOT depend on dataset size, only on sequence length, window width and number of singular vectors. Depending on algorithm being used, incremental update has complexity: * for top-K real time - O(UpdateIts*K*Width^2+(NTicks-Width)*Width^2) * for top-K direct - O(Width^3+(NTicks-Width)*Width^2) * for precomputed basis - O(1), no update is performed INPUT PARAMETERS: S - SSA model created with ssacreate() X - new sequence, array[NTicks] or larget NTicks - >=1, number of ticks in the sequence UpdateIts - >=0, floating point (!) value, desired update frequency: * zero value means that point is stored, but no update is performed * integer part of the value means that specified number of iterations is always performed * fractional part of the value means that one iteration is performed with this probability. Recommended value: 0=0, "SSAAppendSequenceAndUpdate: NTicks<0"); alglib.ap.assert(alglib.ap.len(x)>=nticks, "SSAAppendSequenceAndUpdate: X is too short"); alglib.ap.assert(apserv.isfinitevector(x, nticks, _params), "SSAAppendSequenceAndUpdate: X contains infinities NANs"); // // Add sequence // apserv.ivectorgrowto(ref s.sequenceidx, s.nsequences+2, _params); s.sequenceidx[s.nsequences+1] = s.sequenceidx[s.nsequences]+nticks; apserv.rvectorgrowto(ref s.sequencedata, s.sequenceidx[s.nsequences+1], _params); offs = s.sequenceidx[s.nsequences]; for(i=0; i<=nticks-1; i++) { s.sequencedata[offs+i] = x[i]; } apserv.inc(ref s.nsequences, _params); // // Do we have something to analyze? If no, invalidate basis // (just to be sure) and exit. // if( !hassomethingtoanalyze(s, _params) ) { s.arebasisandsolvervalid = false; return; } // // Well, we have data to analyze and algorithm set, but basis is // invalid. Let's calculate it from scratch and exit. // if( !s.arebasisandsolvervalid ) { updatebasis(s, 0, 0.0, _params); return; } // // Update already computed basis // if( nticks>=s.windowwidth ) { updatebasis(s, nticks-s.windowwidth+1, updateits, _params); } } /************************************************************************* This function sets SSA algorithm to "precomputed vectors" algorithm. This algorithm uses precomputed set of orthonormal (orthogonal AND normalized) basis vectors supplied by user. Thus, basis calculation phase is not performed - we already have our basis - and only analysis/ forecasting phase requires actual calculations. This algorithm may handle "append" requests which add just one/few ticks to the end of the last sequence in O(1) time. NOTE: this algorithm accepts both basis and window width, because these two parameters are naturally aligned. Calling this function sets window width; if you call ssasetwindow() with other window width, then during analysis stage algorithm will detect conflict and reset to zero basis. INPUT PARAMETERS: S - SSA model A - array[WindowWidth,NBasis], orthonormalized basis; this function does NOT control orthogonality and does NOT perform any kind of renormalization. It is your responsibility to provide it with correct basis. WindowWidth - window width, >=1 NBasis - number of basis vectors, 1<=NBasis<=WindowWidth OUTPUT PARAMETERS: S - updated model NOTE: calling this function invalidates basis in all cases. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetalgoprecomputed(ssamodel s, double[,] a, int windowwidth, int nbasis, alglib.xparams _params) { int i = 0; int j = 0; alglib.ap.assert(windowwidth>=1, "SSASetAlgoPrecomputed: WindowWidth<1"); alglib.ap.assert(nbasis>=1, "SSASetAlgoPrecomputed: NBasis<1"); alglib.ap.assert(nbasis<=windowwidth, "SSASetAlgoPrecomputed: NBasis>WindowWidth"); alglib.ap.assert(alglib.ap.rows(a)>=windowwidth, "SSASetAlgoPrecomputed: Rows(A)=nbasis, "SSASetAlgoPrecomputed: Rows(A)=1. OUTPUT PARAMETERS: S - updated model NOTE: TopK>WindowWidth is silently decreased to WindowWidth during analysis phase NOTE: calling this function invalidates basis, except for the situation when this algorithm was already set with same parameters. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetalgotopkdirect(ssamodel s, int topk, alglib.xparams _params) { alglib.ap.assert(topk>=1, "SSASetAlgoTopKDirect: TopK<1"); // // Ignore calls which change nothing // if( s.algotype==2 && s.topk==topk ) { return; } // // Update settings, invalidate model // s.algotype = 2; s.topk = topk; s.arebasisandsolvervalid = false; } /************************************************************************* This function sets SSA algorithm to "top-K real time algorithm". This algo extracts K components with largest singular values. It is real-time version of top-K algorithm which is optimized for incremental processing and fast start-up. Internally it uses subspace eigensolver for truncated SVD. It results in ability to perform quick updates of the basis when only a few points/sequences is added to dataset. Performance profile of the algorithm is given below: * O(K*WindowWidth^2) running time for incremental update of the dataset with one of the "append-and-update" functions (ssaappendpointandupdate() or ssaappendsequenceandupdate()). * O(N*WindowWidth^2) running time for initial basis evaluation (N=size of dataset) * ability to split costly initialization across several incremental updates of the basis (so called "Power-Up" functionality, activated by ssasetpoweruplength() function) INPUT PARAMETERS: S - SSA model TopK - number of components to analyze; TopK>=1. OUTPUT PARAMETERS: S - updated model NOTE: this algorithm is optimized for large-scale tasks with large datasets. On toy problems with just 5-10 points it can return basis which is slightly different from that returned by direct algorithm (ssasetalgotopkdirect() function). However, the difference becomes negligible as dataset grows. NOTE: TopK>WindowWidth is silently decreased to WindowWidth during analysis phase NOTE: calling this function invalidates basis, except for the situation when this algorithm was already set with same parameters. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssasetalgotopkrealtime(ssamodel s, int topk, alglib.xparams _params) { alglib.ap.assert(topk>=1, "SSASetAlgoTopKRealTime: TopK<1"); // // Ignore calls which change nothing // if( s.algotype==3 && s.topk==topk ) { return; } // // Update settings, invalidate model // s.algotype = 3; s.topk = topk; s.arebasisandsolvervalid = false; } /************************************************************************* This function clears all data stored in the model and invalidates all basis components found so far. INPUT PARAMETERS: S - SSA model created with ssacreate() OUTPUT PARAMETERS: S - SSA model, updated -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssacleardata(ssamodel s, alglib.xparams _params) { s.nsequences = 0; s.arebasisandsolvervalid = false; } /************************************************************************* This function executes SSA on internally stored dataset and returns basis found by current method. INPUT PARAMETERS: S - SSA model OUTPUT PARAMETERS: A - array[WindowWidth,NBasis], basis; vectors are stored in matrix columns, by descreasing variance SV - array[NBasis]: * zeros - for model initialized with SSASetAlgoPrecomputed() * singular values - for other algorithms WindowWidth - current window NBasis - basis size CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Calling this function in degenerate cases (no data or all data are shorter than window size; no algorithm is specified) returns basis with just one zero vector. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssagetbasis(ssamodel s, ref double[,] a, ref double[] sv, ref int windowwidth, ref int nbasis, alglib.xparams _params) { int i = 0; a = new double[0,0]; sv = new double[0]; windowwidth = 0; nbasis = 0; // // Is it degenerate case? // if( !hassomethingtoanalyze(s, _params) ) { windowwidth = s.windowwidth; nbasis = 1; a = new double[windowwidth, 1]; for(i=0; i<=windowwidth-1; i++) { a[i,0] = 0.0; } sv = new double[1]; sv[0] = 0.0; return; } // // Update basis. // // It will take care of basis validity flags. AppendLen=0 which means // that we perform initial basis evaluation. // updatebasis(s, 0, 0.0, _params); // // Output // alglib.ap.assert(s.nbasis>0, "SSAGetBasis: integrity check failed"); alglib.ap.assert(s.windowwidth>0, "SSAGetBasis: integrity check failed"); nbasis = s.nbasis; windowwidth = s.windowwidth; a = new double[windowwidth, nbasis]; ablas.rmatrixcopy(windowwidth, nbasis, s.basis, 0, 0, a, 0, 0, _params); sv = new double[nbasis]; for(i=0; i<=nbasis-1; i++) { sv[i] = s.sv[i]; } } /************************************************************************* This function returns linear recurrence relation (LRR) coefficients found by current SSA algorithm. INPUT PARAMETERS: S - SSA model OUTPUT PARAMETERS: A - array[WindowWidth-1]. Coefficients of the linear recurrence of the form: X[W-1] = X[W-2]*A[W-2] + X[W-3]*A[W-3] + ... + X[0]*A[0]. Empty array for WindowWidth=1. WindowWidth - current window width CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Calling this function in degenerate cases (no data or all data are shorter than window size; no algorithm is specified) returns zeros. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssagetlrr(ssamodel s, ref double[] a, ref int windowwidth, alglib.xparams _params) { int i = 0; a = new double[0]; windowwidth = 0; alglib.ap.assert(s.windowwidth>0, "SSAGetLRR: integrity check failed"); // // Is it degenerate case? // if( !hassomethingtoanalyze(s, _params) ) { windowwidth = s.windowwidth; a = new double[windowwidth-1]; for(i=0; i<=windowwidth-2; i++) { a[i] = 0.0; } return; } // // Update basis. // // It will take care of basis validity flags. AppendLen=0 which means // that we perform initial basis evaluation. // updatebasis(s, 0, 0.0, _params); // // Output // windowwidth = s.windowwidth; a = new double[windowwidth-1]; for(i=0; i<=windowwidth-2; i++) { a[i] = s.forecasta[i]; } } /************************************************************************* This function executes SSA on internally stored dataset and returns analysis for the last window of the last sequence. Such analysis is an lightweight alternative for full scale reconstruction (see below). Typical use case for this function is real-time setting, when you are interested in quick-and-dirty (very quick and very dirty) processing of just a few last ticks of the trend. IMPORTANT: full scale SSA involves analysis of the ENTIRE dataset, with reconstruction being done for all positions of sliding window with subsequent hankelization (diagonal averaging) of the resulting matrix. Such analysis requires O((DataLen-Window)*Window*NBasis) FLOPs and can be quite costly. However, it has nice noise-canceling effects due to averaging. This function performs REDUCED analysis of the last window. It is much faster - just O(Window*NBasis), but its results are DIFFERENT from that of ssaanalyzelast(). In particular, first few points of the trend are much more prone to noise. INPUT PARAMETERS: S - SSA model OUTPUT PARAMETERS: Trend - array[WindowSize], reconstructed trend line Noise - array[WindowSize], the rest of the signal; it holds that ActualData = Trend+Noise. NTicks - current WindowSize CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. In any case, only basis is reused. Reconstruction is performed from scratch every time you call this function. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * last sequence is shorter than the window length (analysis can be done, but we can not perform reconstruction on the last sequence) Calling this function in degenerate cases returns following result: * in any case, WindowWidth ticks is returned * trend is assumed to be zero * noise is initialized by the last sequence; if last sequence is shorter than the window size, it is moved to the end of the array, and the beginning of the noise array is filled by zeros No analysis is performed in degenerate cases (we immediately return dummy values, no basis is constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaanalyzelastwindow(ssamodel s, ref double[] trend, ref double[] noise, ref int nticks, alglib.xparams _params) { int i = 0; int offs = 0; int cnt = 0; trend = new double[0]; noise = new double[0]; nticks = 0; // // Init // nticks = s.windowwidth; trend = new double[s.windowwidth]; noise = new double[s.windowwidth]; // // Is it degenerate case? // if( !hassomethingtoanalyze(s, _params) || !issequencebigenough(s, -1, _params) ) { for(i=0; i<=nticks-1; i++) { trend[i] = 0; noise[i] = 0; } if( s.nsequences>=1 ) { cnt = Math.Min(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1], nticks); offs = s.sequenceidx[s.nsequences]-cnt; for(i=0; i<=cnt-1; i++) { noise[nticks-cnt+i] = s.sequencedata[offs+i]; } } return; } // // Update basis. // // It will take care of basis validity flags. AppendLen=0 which means // that we perform initial basis evaluation. // updatebasis(s, 0, 0.0, _params); // // Perform analysis of the last window // alglib.ap.assert(s.sequenceidx[s.nsequences]-s.windowwidth>=0, "SSAAnalyzeLastWindow: integrity check failed"); apserv.rvectorsetlengthatleast(ref s.tmp0, s.nbasis, _params); ablas.rmatrixgemv(s.nbasis, s.windowwidth, 1.0, s.basist, 0, 0, 0, s.sequencedata, s.sequenceidx[s.nsequences]-s.windowwidth, 0.0, s.tmp0, 0, _params); ablas.rmatrixgemv(s.windowwidth, s.nbasis, 1.0, s.basis, 0, 0, 0, s.tmp0, 0, 0.0, trend, 0, _params); offs = s.sequenceidx[s.nsequences]-s.windowwidth; cnt = s.windowwidth; for(i=0; i<=cnt-1; i++) { noise[i] = s.sequencedata[offs+i]-trend[i]; } } /************************************************************************* This function: * builds SSA basis using internally stored (entire) dataset * returns reconstruction for the last NTicks of the last sequence If you want to analyze some other sequence, use ssaanalyzesequence(). Reconstruction phase involves generation of NTicks-WindowWidth sliding windows, their decomposition using empirical orthogonal functions found by SSA, followed by averaging of each data point across several overlapping windows. Thus, every point in the output trend is reconstructed using up to WindowWidth overlapping windows (WindowWidth windows exactly in the inner points, just one window at the extremal points). IMPORTANT: due to averaging this function returns different results for different values of NTicks. It is expected and not a bug. For example: * Trend[NTicks-1] is always same because it is not averaged in any case (same applies to Trend[0]). * Trend[NTicks-2] has different values for NTicks=WindowWidth and NTicks=WindowWidth+1 because former case means that no averaging is performed, and latter case means that averaging using two sliding windows is performed. Larger values of NTicks produce same results as NTicks=WindowWidth+1. * ...and so on... PERFORMANCE: this function has O((NTicks-WindowWidth)*WindowWidth*NBasis) running time. If you work in time-constrained setting and have to analyze just a few last ticks, choosing NTicks equal to WindowWidth+SmoothingLen, with SmoothingLen=1...WindowWidth will result in good compromise between noise cancellation and analysis speed. INPUT PARAMETERS: S - SSA model NTicks - number of ticks to analyze, Nticks>=1. * special case of NTicks<=WindowWidth is handled by analyzing last window and returning NTicks last ticks. * special case NTicks>LastSequenceLen is handled by prepending result with NTicks-LastSequenceLen zeros. OUTPUT PARAMETERS: Trend - array[NTicks], reconstructed trend line Noise - array[NTicks], the rest of the signal; it holds that ActualData = Trend+Noise. CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. In any case, only basis is reused. Reconstruction is performed from scratch every time you call this function. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * last sequence is shorter than the window length (analysis can be done, but we can not perform reconstruction on the last sequence) Calling this function in degenerate cases returns following result: * in any case, NTicks ticks is returned * trend is assumed to be zero * noise is initialized by the last sequence; if last sequence is shorter than the window size, it is moved to the end of the array, and the beginning of the noise array is filled by zeros No analysis is performed in degenerate cases (we immediately return dummy values, no basis is constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaanalyzelast(ssamodel s, int nticks, ref double[] trend, ref double[] noise, alglib.xparams _params) { int i = 0; int offs = 0; int cnt = 0; int cntzeros = 0; trend = new double[0]; noise = new double[0]; alglib.ap.assert(nticks>=1, "SSAAnalyzeLast: NTicks<1"); // // Init // trend = new double[nticks]; noise = new double[nticks]; // // Is it degenerate case? // if( !hassomethingtoanalyze(s, _params) || !issequencebigenough(s, -1, _params) ) { for(i=0; i<=nticks-1; i++) { trend[i] = 0; noise[i] = 0; } if( s.nsequences>=1 ) { cnt = Math.Min(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1], nticks); offs = s.sequenceidx[s.nsequences]-cnt; for(i=0; i<=cnt-1; i++) { noise[nticks-cnt+i] = s.sequencedata[offs+i]; } } return; } // // Fast exit: NTicks<=WindowWidth, just last window is analyzed // if( nticks<=s.windowwidth ) { ssaanalyzelastwindow(s, ref s.alongtrend, ref s.alongnoise, ref cnt, _params); offs = s.windowwidth-nticks; for(i=0; i<=nticks-1; i++) { trend[i] = s.alongtrend[offs+i]; noise[i] = s.alongnoise[offs+i]; } return; } // // Update basis. // // It will take care of basis validity flags. AppendLen=0 which means // that we perform initial basis evaluation. // updatebasis(s, 0, 0.0, _params); // // Perform analysis: // * prepend max(NTicks-LastSequenceLength,0) zeros to the beginning // of array // * analyze the rest with AnalyzeSequence() which assumes that we // already have basis // alglib.ap.assert(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]>=s.windowwidth, "SSAAnalyzeLast: integrity check failed / 23vd4"); cntzeros = Math.Max(nticks-(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]), 0); for(i=0; i<=cntzeros-1; i++) { trend[i] = 0.0; noise[i] = 0.0; } cnt = Math.Min(nticks, s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]); analyzesequence(s, s.sequencedata, s.sequenceidx[s.nsequences]-cnt, s.sequenceidx[s.nsequences], trend, noise, cntzeros, _params); } /************************************************************************* This function: * builds SSA basis using internally stored (entire) dataset * returns reconstruction for the sequence being passed to this function If you want to analyze last sequence stored in the model, use ssaanalyzelast(). Reconstruction phase involves generation of NTicks-WindowWidth sliding windows, their decomposition using empirical orthogonal functions found by SSA, followed by averaging of each data point across several overlapping windows. Thus, every point in the output trend is reconstructed using up to WindowWidth overlapping windows (WindowWidth windows exactly in the inner points, just one window at the extremal points). PERFORMANCE: this function has O((NTicks-WindowWidth)*WindowWidth*NBasis) running time. If you work in time-constrained setting and have to analyze just a few last ticks, choosing NTicks equal to WindowWidth+SmoothingLen, with SmoothingLen=1...WindowWidth will result in good compromise between noise cancellation and analysis speed. INPUT PARAMETERS: S - SSA model Data - array[NTicks], can be larger (only NTicks leading elements will be used) NTicks - number of ticks to analyze, Nticks>=1. * special case of NTicks=1, "SSAAnalyzeSequence: NTicks<1"); alglib.ap.assert(alglib.ap.len(data)>=nticks, "SSAAnalyzeSequence: Data is too short"); alglib.ap.assert(apserv.isfinitevector(data, nticks, _params), "SSAAnalyzeSequence: Data contains infinities NANs"); // // Init // trend = new double[nticks]; noise = new double[nticks]; // // Is it degenerate case? // if( !hassomethingtoanalyze(s, _params) || nticks=1 OUTPUT PARAMETERS: Trend - array[NTicks], predicted trend line CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * last sequence is shorter than the WindowWidth (analysis can be done, but we can not perform forecasting on the last sequence) * window lentgh is 1 (impossible to use for forecasting) * SSA analysis algorithm is configured to extract basis whose size is equal to window length (impossible to use for forecasting; only basis whose size is less than window length can be used). Calling this function in degenerate cases returns following result: * NTicks copies of the last value is returned for non-empty task with large enough dataset, but with overcomplete basis (window width=1 or basis size is equal to window width) * zero trend with length=NTicks is returned for empty task No analysis is performed in degenerate cases (we immediately return dummy values, no basis is ever constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaforecastlast(ssamodel s, int nticks, ref double[] trend, alglib.xparams _params) { int i = 0; int j = 0; double v = 0; int winw = 0; trend = new double[0]; alglib.ap.assert(nticks>=1, "SSAForecast: NTicks<1"); // // Init // winw = s.windowwidth; trend = new double[nticks]; // // Is it degenerate case? // if( !hassomethingtoanalyze(s, _params) ) { for(i=0; i<=nticks-1; i++) { trend[i] = 0; } return; } alglib.ap.assert(s.nsequences>0, "SSAForecastLast: integrity check failed"); if( s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]0, "SSAForecast: integrity check failed / 2355"); alglib.ap.assert(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]>0, "SSAForecast: integrity check failed"); for(i=0; i<=nticks-1; i++) { trend[i] = s.sequencedata[s.sequenceidx[s.nsequences]-1]; } return; } // // Update basis and recurrent relation. // // It will take care of basis validity flags. AppendLen=0 which means // that we perform initial basis evaluation. // updatebasis(s, 0, 0.0, _params); alglib.ap.assert(s.nbasis<=winw && s.nbasis>0, "SSAForecast: integrity check failed / 4f5et"); if( s.nbasis==winw ) { // // Handle degenerate situation with basis whose size // is equal to window length. // alglib.ap.assert(s.nsequences>0, "SSAForecast: integrity check failed / 2355"); alglib.ap.assert(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]>0, "SSAForecast: integrity check failed"); for(i=0; i<=nticks-1; i++) { trend[i] = s.sequencedata[s.sequenceidx[s.nsequences]-1]; } return; } // // Apply recurrent formula for SSA forecasting: // * first, perform smoothing of the last window // * second, perform analysis phase // alglib.ap.assert(s.nsequences>0, "SSAForecastLast: integrity check failed"); alglib.ap.assert(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]>=s.windowwidth, "SSAForecastLast: integrity check failed"); apserv.rvectorsetlengthatleast(ref s.tmp0, s.nbasis, _params); apserv.rvectorsetlengthatleast(ref s.fctrend, s.windowwidth, _params); ablas.rmatrixgemv(s.nbasis, s.windowwidth, 1.0, s.basist, 0, 0, 0, s.sequencedata, s.sequenceidx[s.nsequences]-s.windowwidth, 0.0, s.tmp0, 0, _params); ablas.rmatrixgemv(s.windowwidth, s.nbasis, 1.0, s.basis, 0, 0, 0, s.tmp0, 0, 0.0, s.fctrend, 0, _params); apserv.rvectorsetlengthatleast(ref s.tmp1, winw-1, _params); for(i=1; i<=winw-1; i++) { s.tmp1[i-1] = s.fctrend[i]; } for(i=0; i<=nticks-1; i++) { v = s.forecasta[0]*s.tmp1[0]; for(j=1; j<=winw-2; j++) { v = v+s.forecasta[j]*s.tmp1[j]; s.tmp1[j-1] = s.tmp1[j]; } trend[i] = v; s.tmp1[winw-2] = v; } } /************************************************************************* This function builds SSA basis and performs forecasting for a user- specified sequence, returning value of trend. Forecasting is done in two stages: * first, we extract trend from the WindowWidth last elements of the sequence. This stage is optional, you can turn it off if you pass data which are already processed with SSA. Of course, you can turn it off even for raw data, but it is not recommended - noise suppression is very important for correct prediction. * then, we apply LRR for last WindowWidth-1 elements of the extracted trend. This function has following running time: * O(NBasis*WindowWidth) for trend extraction phase * O(WindowWidth*NTicks) for forecast phase NOTE: this algorithm performs prediction using only one - last - sliding window. Predictions produced by such approach are smooth continuations of the reconstructed trend line, but they can be easily corrupted by noise. If you need noise-resistant prediction, use ssaforecastavgsequence() function, which averages predictions built using several sliding windows. INPUT PARAMETERS: S - SSA model Data - array[NTicks], data to forecast DataLen - number of ticks in the data, DataLen>=1 ForecastLen - number of ticks to predict, ForecastLen>=1 ApplySmoothing - whether to apply smoothing trend extraction or not; if you do not know what to specify, pass True. OUTPUT PARAMETERS: Trend - array[ForecastLen], forecasted trend CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * data sequence is shorter than the WindowWidth (analysis can be done, but we can not perform forecasting on the last sequence) * window lentgh is 1 (impossible to use for forecasting) * SSA analysis algorithm is configured to extract basis whose size is equal to window length (impossible to use for forecasting; only basis whose size is less than window length can be used). Calling this function in degenerate cases returns following result: * ForecastLen copies of the last value is returned for non-empty task with large enough dataset, but with overcomplete basis (window width=1 or basis size is equal to window width) * zero trend with length=ForecastLen is returned for empty task No analysis is performed in degenerate cases (we immediately return dummy values, no basis is ever constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaforecastsequence(ssamodel s, double[] data, int datalen, int forecastlen, bool applysmoothing, ref double[] trend, alglib.xparams _params) { int i = 0; int j = 0; double v = 0; int winw = 0; trend = new double[0]; alglib.ap.assert(datalen>=1, "SSAForecastSequence: DataLen<1"); alglib.ap.assert(alglib.ap.len(data)>=datalen, "SSAForecastSequence: Data is too short"); alglib.ap.assert(apserv.isfinitevector(data, datalen, _params), "SSAForecastSequence: Data contains infinities NANs"); alglib.ap.assert(forecastlen>=1, "SSAForecastSequence: ForecastLen<1"); // // Init // winw = s.windowwidth; trend = new double[forecastlen]; // // Is it degenerate case? // if( !hassomethingtoanalyze(s, _params) || datalen0, "SSAForecast: integrity check failed / 4f5et"); if( s.nbasis==winw ) { // // Handle degenerate situation with basis whose size // is equal to window length. // for(i=0; i<=forecastlen-1; i++) { trend[i] = data[datalen-1]; } return; } // // Perform trend extraction // apserv.rvectorsetlengthatleast(ref s.fctrend, s.windowwidth, _params); if( applysmoothing ) { alglib.ap.assert(datalen>=winw, "SSAForecastSequence: integrity check failed"); apserv.rvectorsetlengthatleast(ref s.tmp0, s.nbasis, _params); ablas.rmatrixgemv(s.nbasis, winw, 1.0, s.basist, 0, 0, 0, data, datalen-winw, 0.0, s.tmp0, 0, _params); ablas.rmatrixgemv(winw, s.nbasis, 1.0, s.basis, 0, 0, 0, s.tmp0, 0, 0.0, s.fctrend, 0, _params); } else { for(i=0; i<=winw-1; i++) { s.fctrend[i] = data[datalen+i-winw]; } } // // Apply recurrent formula for SSA forecasting // apserv.rvectorsetlengthatleast(ref s.tmp1, winw-1, _params); for(i=1; i<=winw-1; i++) { s.tmp1[i-1] = s.fctrend[i]; } for(i=0; i<=forecastlen-1; i++) { v = s.forecasta[0]*s.tmp1[0]; for(j=1; j<=winw-2; j++) { v = v+s.forecasta[j]*s.tmp1[j]; s.tmp1[j-1] = s.tmp1[j]; } trend[i] = v; s.tmp1[winw-2] = v; } } /************************************************************************* This function builds SSA basis and performs forecasting for a specified number of ticks, returning value of trend. Forecast is performed as follows: * SSA trend extraction is applied to last M sliding windows of the internally stored dataset * for each of M sliding windows, M predictions are built * average value of M predictions is returned This function has following running time: * O(NBasis*WindowWidth*M) for trend extraction phase (always performed) * O(WindowWidth*NTicks*M) for forecast phase NOTE: noise reduction is ALWAYS applied by this algorithm; if you want to apply recurrence relation to raw unprocessed data, use another function - ssaforecastsequence() which allows to turn on and off noise reduction phase. NOTE: combination of several predictions results in lesser sensitivity to noise, but it may produce undesirable discontinuities between last point of the trend and first point of the prediction. The reason is that last point of the trend is usually corrupted by noise, but average value of several predictions is less sensitive to noise, thus discontinuity appears. It is not a bug. INPUT PARAMETERS: S - SSA model M - number of sliding windows to combine, M>=1. If your dataset has less than M sliding windows, this parameter will be silently reduced. NTicks - number of ticks to forecast, NTicks>=1 OUTPUT PARAMETERS: Trend - array[NTicks], predicted trend line CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * last sequence is shorter than the WindowWidth (analysis can be done, but we can not perform forecasting on the last sequence) * window lentgh is 1 (impossible to use for forecasting) * SSA analysis algorithm is configured to extract basis whose size is equal to window length (impossible to use for forecasting; only basis whose size is less than window length can be used). Calling this function in degenerate cases returns following result: * NTicks copies of the last value is returned for non-empty task with large enough dataset, but with overcomplete basis (window width=1 or basis size is equal to window width) * zero trend with length=NTicks is returned for empty task No analysis is performed in degenerate cases (we immediately return dummy values, no basis is ever constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaforecastavglast(ssamodel s, int m, int nticks, ref double[] trend, alglib.xparams _params) { int i = 0; int winw = 0; trend = new double[0]; alglib.ap.assert(nticks>=1, "SSAForecastAvgLast: NTicks<1"); alglib.ap.assert(m>=1, "SSAForecastAvgLast: M<1"); // // Init // winw = s.windowwidth; trend = new double[nticks]; // // Is it degenerate case? // if( !hassomethingtoanalyze(s, _params) ) { for(i=0; i<=nticks-1; i++) { trend[i] = 0; } return; } alglib.ap.assert(s.nsequences>0, "SSAForecastAvgLast: integrity check failed"); if( s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]0, "SSAForecastAvgLast: integrity check failed / 2355"); alglib.ap.assert(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]>0, "SSAForecastAvgLast: integrity check failed"); for(i=0; i<=nticks-1; i++) { trend[i] = s.sequencedata[s.sequenceidx[s.nsequences]-1]; } return; } // // Update basis and recurrent relation. // // It will take care of basis validity flags. AppendLen=0 which means // that we perform initial basis evaluation. // updatebasis(s, 0, 0.0, _params); alglib.ap.assert(s.nbasis<=winw && s.nbasis>0, "SSAForecastAvgLast: integrity check failed / 4f5et"); if( s.nbasis==winw ) { // // Handle degenerate situation with basis whose size // is equal to window length. // alglib.ap.assert(s.nsequences>0, "SSAForecastAvgLast: integrity check failed / 2355"); alglib.ap.assert(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]>0, "SSAForecastAvgLast: integrity check failed"); for(i=0; i<=nticks-1; i++) { trend[i] = s.sequencedata[s.sequenceidx[s.nsequences]-1]; } return; } // // Decrease M if we have less than M sliding windows. // Forecast. // m = Math.Min(m, s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]-winw+1); alglib.ap.assert(m>=1, "SSAForecastAvgLast: integrity check failed"); forecastavgsequence(s, s.sequencedata, s.sequenceidx[s.nsequences-1], s.sequenceidx[s.nsequences], m, nticks, true, trend, 0, _params); } /************************************************************************* This function builds SSA basis and performs forecasting for a user- specified sequence, returning value of trend. Forecasting is done in two stages: * first, we extract trend from M last sliding windows of the sequence. This stage is optional, you can turn it off if you pass data which are already processed with SSA. Of course, you can turn it off even for raw data, but it is not recommended - noise suppression is very important for correct prediction. * then, we apply LRR independently for M sliding windows * average of M predictions is returned This function has following running time: * O(NBasis*WindowWidth*M) for trend extraction phase * O(WindowWidth*NTicks*M) for forecast phase NOTE: combination of several predictions results in lesser sensitivity to noise, but it may produce undesirable discontinuities between last point of the trend and first point of the prediction. The reason is that last point of the trend is usually corrupted by noise, but average value of several predictions is less sensitive to noise, thus discontinuity appears. It is not a bug. INPUT PARAMETERS: S - SSA model Data - array[NTicks], data to forecast DataLen - number of ticks in the data, DataLen>=1 M - number of sliding windows to combine, M>=1. If your dataset has less than M sliding windows, this parameter will be silently reduced. ForecastLen - number of ticks to predict, ForecastLen>=1 ApplySmoothing - whether to apply smoothing trend extraction or not. if you do not know what to specify, pass true. OUTPUT PARAMETERS: Trend - array[ForecastLen], forecasted trend CACHING/REUSE OF THE BASIS Caching/reuse of previous results is performed: * first call performs full run of SSA; basis is stored in the cache * subsequent calls reuse previously cached basis * if you call any function which changes model properties (window length, algorithm, dataset), internal basis will be invalidated. * the only calls which do NOT invalidate basis are listed below: a) ssasetwindow() with same window length b) ssaappendpointandupdate() c) ssaappendsequenceandupdate() d) ssasetalgotopk...() with exactly same K Calling these functions will result in reuse of previously found basis. HANDLING OF DEGENERATE CASES Following degenerate cases may happen: * dataset is empty (no analysis can be done) * all sequences are shorter than the window length,no analysis can be done * no algorithm is specified (no analysis can be done) * data sequence is shorter than the WindowWidth (analysis can be done, but we can not perform forecasting on the last sequence) * window lentgh is 1 (impossible to use for forecasting) * SSA analysis algorithm is configured to extract basis whose size is equal to window length (impossible to use for forecasting; only basis whose size is less than window length can be used). Calling this function in degenerate cases returns following result: * ForecastLen copies of the last value is returned for non-empty task with large enough dataset, but with overcomplete basis (window width=1 or basis size is equal to window width) * zero trend with length=ForecastLen is returned for empty task No analysis is performed in degenerate cases (we immediately return dummy values, no basis is ever constructed). -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ public static void ssaforecastavgsequence(ssamodel s, double[] data, int datalen, int m, int forecastlen, bool applysmoothing, ref double[] trend, alglib.xparams _params) { int i = 0; int winw = 0; trend = new double[0]; alglib.ap.assert(datalen>=1, "SSAForecastAvgSequence: DataLen<1"); alglib.ap.assert(m>=1, "SSAForecastAvgSequence: M<1"); alglib.ap.assert(alglib.ap.len(data)>=datalen, "SSAForecastAvgSequence: Data is too short"); alglib.ap.assert(apserv.isfinitevector(data, datalen, _params), "SSAForecastAvgSequence: Data contains infinities NANs"); alglib.ap.assert(forecastlen>=1, "SSAForecastAvgSequence: ForecastLen<1"); // // Init // winw = s.windowwidth; trend = new double[forecastlen]; // // Is it degenerate case? // if( !hassomethingtoanalyze(s, _params) || datalen0, "SSAForecast: integrity check failed / 4f5et"); if( s.nbasis==winw ) { // // Handle degenerate situation with basis whose size // is equal to window length. // for(i=0; i<=forecastlen-1; i++) { trend[i] = data[datalen-1]; } return; } // // Decrease M if we have less than M sliding windows. // Forecast. // m = Math.Min(m, datalen-winw+1); alglib.ap.assert(m>=1, "SSAForecastAvgLast: integrity check failed"); forecastavgsequence(s, data, 0, datalen, m, forecastlen, applysmoothing, trend, 0, _params); } /************************************************************************* This function evaluates current model and tells whether we have some data which can be analyzed by current algorithm, or not. No analysis can be done in the following degenerate cases: * dataset is empty * all sequences are shorter than the window length * no algorithm is specified -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ private static bool hassomethingtoanalyze(ssamodel s, alglib.xparams _params) { bool result = new bool(); int i = 0; bool allsmaller = new bool(); bool isdegenerate = new bool(); isdegenerate = false; isdegenerate = isdegenerate || s.algotype==0; isdegenerate = isdegenerate || s.nsequences==0; allsmaller = true; for(i=0; i<=s.nsequences-1; i++) { allsmaller = allsmaller && s.sequenceidx[i+1]-s.sequenceidx[i]=-1 && i=s.windowwidth; return result; } /************************************************************************* This function performs basis update. Either full update (recalculated from the very beginning) or partial update (handles append to the end of the dataset). With AppendLen=0 this function behaves as follows: * if AreBasisAndSolverValid=False, then solver object is created from scratch, initial calculations are performed according to specific SSA algorithm being chosen. Basis/Solver validity flag is set to True, then we immediately return. * if AreBasisAndSolverValid=True, then nothing is done - we immediately return. With AppendLen>0 this function behaves as follows: * if AreBasisAndSolverValid=False, then exception is generated; you can append points only to fully constructed basis. Call this function with zero AppendLen BEFORE append, then perform append, then call it one more time with non-zero AppendLen. * if AreBasisAndSolverValid=True, then basis is incrementally updated. It also updates recurrence relation used for prediction. It is expected that either AppendLen=1, or AppendLen=length(last_sequence). Basis update is performed with probability UpdateIts (larger-than-one values mean that some amount of iterations is always performed). In any case, after calling this function we either: * have an exception * have completely valid basis IMPORTANT: this function expects that we do NOT call it for degenerate tasks (no data). So, call it after check with HasSomethingToAnalyze() returned True. -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ private static void updatebasis(ssamodel s, int appendlen, double updateits, alglib.xparams _params) { int i = 0; int j = 0; int k = 0; int srcoffs = 0; int dstoffs = 0; int winw = 0; int windowstotal = 0; int requesttype = 0; int requestsize = 0; double v = 0; bool degeneraterecurrence = new bool(); double nu2 = 0; int subspaceits = 0; bool needevd = new bool(); winw = s.windowwidth; // // Critical checks // alglib.ap.assert(appendlen>=0, "SSA: incorrect parameters passed to UpdateBasis(), integrity check failed"); alglib.ap.assert(!(!s.arebasisandsolvervalid && appendlen!=0), "SSA: incorrect parameters passed to UpdateBasis(), integrity check failed"); alglib.ap.assert(!(appendlen==0 && (double)(updateits)>(double)(0.0)), "SSA: incorrect parameters passed to UpdateBasis(), integrity check failed"); // // Everything is OK, nothing to do // if( s.arebasisandsolvervalid && appendlen==0 ) { return; } // // Seed RNG with fixed or random seed. // // RNG used when pseudorandomly deciding whether // to re-evaluate basis or not. Sandom seed is // important when we have several simultaneously // calculated SSA models - we do not want them // to be re-evaluated in same moments). // if( !s.arebasisandsolvervalid ) { if( s.rngseed>0 ) { hqrnd.hqrndseed(s.rngseed, s.rngseed+235, s.rs, _params); } else { hqrnd.hqrndrandomize(s.rs, _params); } } // // Compute XXT for algorithms which need it // if( !s.arebasisandsolvervalid ) { alglib.ap.assert(appendlen==0, "SSA: integrity check failed / 34cx6"); if( s.algotype==2 ) { // // Compute X*X^T for direct algorithm. // Quite straightforward, no subtle optimizations. // apserv.rmatrixsetlengthatleast(ref s.xxt, winw, winw, _params); windowstotal = 0; for(i=0; i<=s.nsequences-1; i++) { windowstotal = windowstotal+Math.Max(s.sequenceidx[i+1]-s.sequenceidx[i]-winw+1, 0); } alglib.ap.assert(windowstotal>0, "SSA: integrity check in UpdateBasis() failed / 76t34"); for(i=0; i<=winw-1; i++) { for(j=0; j<=winw-1; j++) { s.xxt[i,j] = 0; } } updatexxtprepare(s, windowstotal, winw, s.memorylimit, _params); for(i=0; i<=s.nsequences-1; i++) { for(j=0; j<=Math.Max(s.sequenceidx[i+1]-s.sequenceidx[i]-winw+1, 0)-1; j++) { updatexxtsend(s, s.sequencedata, s.sequenceidx[i]+j, s.xxt, _params); } } updatexxtfinalize(s, s.xxt, _params); } if( s.algotype==3 ) { // // Compute X*X^T for real-time algorithm: // * prepare queue of windows to merge into XXT // * shuffle queue in order to avoid time-related biases in algorithm // * dequeue first chunk // apserv.rmatrixsetlengthatleast(ref s.xxt, winw, winw, _params); windowstotal = 0; for(i=0; i<=s.nsequences-1; i++) { windowstotal = windowstotal+Math.Max(s.sequenceidx[i+1]-s.sequenceidx[i]-winw+1, 0); } alglib.ap.assert(windowstotal>0, "SSA: integrity check in UpdateBasis() failed / 76t34"); apserv.ivectorsetlengthatleast(ref s.rtqueue, windowstotal, _params); dstoffs = 0; for(i=0; i<=s.nsequences-1; i++) { for(j=0; j<=Math.Max(s.sequenceidx[i+1]-s.sequenceidx[i]-winw+1, 0)-1; j++) { srcoffs = s.sequenceidx[i]+j; s.rtqueue[dstoffs] = srcoffs; apserv.inc(ref dstoffs, _params); } } alglib.ap.assert(dstoffs==windowstotal, "SSA: integrity check in UpdateBasis() failed / fh45f"); if( s.rtpowerup>1 ) { // // Shuffle queue, it helps to avoid time-related bias in algorithm // for(i=0; i<=windowstotal-1; i++) { j = i+hqrnd.hqrnduniformi(s.rs, windowstotal-i, _params); apserv.swapelementsi(s.rtqueue, i, j, _params); } } s.rtqueuecnt = windowstotal; s.rtqueuechunk = 1; s.rtqueuechunk = Math.Max(s.rtqueuechunk, s.rtqueuecnt/s.rtpowerup); s.rtqueuechunk = Math.Max(s.rtqueuechunk, 2*s.topk); realtimedequeue(s, 0.0, Math.Min(s.rtqueuechunk, s.rtqueuecnt), _params); } } // // Handle possible updates for XXT: // * check that append involves either last point of last sequence, // or entire last sequence // * if last sequence is shorter than window width, perform quick exit - // we have nothing to update - no windows to insert into XXT // * update XXT // if( appendlen>0 ) { alglib.ap.assert(s.arebasisandsolvervalid, "SSA: integrity check failed / 5gvz3"); alglib.ap.assert(s.nsequences>=1, "SSA: integrity check failed / 658ev"); alglib.ap.assert(appendlen==1 || appendlen==s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]-winw+1, "SSA: integrity check failed / sd3g7"); if( s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]1 ) { // // Long append, use GEMM for updates // updatexxtprepare(s, appendlen, winw, s.memorylimit, _params); for(j=0; j<=Math.Max(s.sequenceidx[s.nsequences]-s.sequenceidx[s.nsequences-1]-winw+1, 0)-1; j++) { updatexxtsend(s, s.sequencedata, s.sequenceidx[s.nsequences-1]+j, s.xxt, _params); } updatexxtfinalize(s, s.xxt, _params); } else { // // Just one element is added, use rank-1 update // ablas.rmatrixger(winw, winw, s.xxt, 0, 0, 1.0, s.sequencedata, s.sequenceidx[s.nsequences]-winw, s.sequencedata, s.sequenceidx[s.nsequences]-winw, _params); } } } // // Now, perform basis calculation - either full recalculation (AppendLen=0) // or quick update (AppendLen>0). // if( s.algotype==1 ) { // // Precomputed basis // if( winw!=s.precomputedwidth ) { // // Window width has changed, reset basis to zeros // s.nbasis = 1; apserv.rmatrixsetlengthatleast(ref s.basis, winw, 1, _params); apserv.rvectorsetlengthatleast(ref s.sv, 1, _params); for(i=0; i<=winw-1; i++) { s.basis[i,0] = 0.0; } s.sv[0] = 0.0; } else { // // OK, use precomputed basis // s.nbasis = s.precomputednbasis; apserv.rmatrixsetlengthatleast(ref s.basis, winw, s.nbasis, _params); apserv.rvectorsetlengthatleast(ref s.sv, s.nbasis, _params); for(j=0; j<=s.nbasis-1; j++) { s.sv[j] = 0.0; for(i=0; i<=winw-1; i++) { s.basis[i,j] = s.precomputedbasis[i,j]; } } } apserv.rmatrixsetlengthatleast(ref s.basist, s.nbasis, winw, _params); ablas.rmatrixtranspose(winw, s.nbasis, s.basis, 0, 0, s.basist, 0, 0, _params); } else { if( s.algotype==2 ) { // // Direct top-K algorithm // // Calculate eigenvectors with SMatrixEVD(), reorder by descending // of magnitudes. // // Update is performed for invalid basis or for non-zero UpdateIts. // needevd = !s.arebasisandsolvervalid; needevd = needevd || (double)(updateits)>=(double)(1); needevd = needevd || (double)(hqrnd.hqrnduniformr(s.rs, _params))<(double)(updateits-(int)Math.Floor(updateits)); if( needevd ) { apserv.inc(ref s.dbgcntevd, _params); s.nbasis = Math.Min(winw, s.topk); if( !evd.smatrixevd(s.xxt, winw, 1, true, ref s.sv, ref s.basis, _params) ) { alglib.ap.assert(false, "SSA: SMatrixEVD failed"); } for(i=0; i<=winw-1; i++) { k = winw-1-i; if( i>=k ) { break; } v = s.sv[i]; s.sv[i] = s.sv[k]; s.sv[k] = v; for(j=0; j<=winw-1; j++) { v = s.basis[j,i]; s.basis[j,i] = s.basis[j,k]; s.basis[j,k] = v; } } for(i=0; i<=s.nbasis-1; i++) { s.sv[i] = Math.Sqrt(Math.Max(s.sv[i], 0.0)); } apserv.rmatrixsetlengthatleast(ref s.basist, s.nbasis, winw, _params); ablas.rmatrixtranspose(winw, s.nbasis, s.basis, 0, 0, s.basist, 0, 0, _params); } } else { if( s.algotype==3 ) { // // Real-time top-K. // // Determine actual number of basis components, prepare subspace // solver (either create from scratch or reuse). // // Update is always performed for invalid basis; for a valid basis // it is performed with probability UpdateIts. // if( s.rtpowerup==1 ) { subspaceits = s.defaultsubspaceits; } else { subspaceits = 3; } if( appendlen>0 ) { alglib.ap.assert(s.arebasisandsolvervalid, "SSA: integrity check in UpdateBasis() failed / srg6f"); alglib.ap.assert((double)(updateits)>=(double)(0), "SSA: integrity check in UpdateBasis() failed / srg4f"); subspaceits = (int)Math.Floor(updateits); if( (double)(hqrnd.hqrnduniformr(s.rs, _params))<(double)(updateits-(int)Math.Floor(updateits)) ) { apserv.inc(ref subspaceits, _params); } alglib.ap.assert(subspaceits>=0, "SSA: integrity check in UpdateBasis() failed / srg9f"); } // // Dequeue pending dataset and merge it into XXT. // // Dequeuing is done only for appends, and only when we have // non-empty queue. // if( appendlen>0 && s.rtqueuecnt>0 ) { realtimedequeue(s, 1.0, Math.Min(s.rtqueuechunk, s.rtqueuecnt), _params); } // // Now, proceed to solver // if( subspaceits>0 ) { if( appendlen==0 ) { s.nbasis = Math.Min(winw, s.topk); evd.eigsubspacecreatebuf(winw, s.nbasis, s.solver, _params); } else { evd.eigsubspacesetwarmstart(s.solver, true, _params); } evd.eigsubspacesetcond(s.solver, 0.0, subspaceits, _params); // // Perform initial basis estimation // apserv.inc(ref s.dbgcntevd, _params); evd.eigsubspaceoocstart(s.solver, 0, _params); while( evd.eigsubspaceooccontinue(s.solver, _params) ) { evd.eigsubspaceoocgetrequestinfo(s.solver, ref requesttype, ref requestsize, _params); alglib.ap.assert(requesttype==0, "SSA: integrity check in UpdateBasis() failed / 346372"); ablas.rmatrixgemm(winw, requestsize, winw, 1.0, s.xxt, 0, 0, 0, s.solver.x, 0, 0, 0, 0.0, s.solver.ax, 0, 0, _params); } evd.eigsubspaceoocstop(s.solver, ref s.sv, ref s.basis, s.solverrep, _params); for(i=0; i<=s.nbasis-1; i++) { s.sv[i] = Math.Sqrt(Math.Max(s.sv[i], 0.0)); } apserv.rmatrixsetlengthatleast(ref s.basist, s.nbasis, winw, _params); ablas.rmatrixtranspose(winw, s.nbasis, s.basis, 0, 0, s.basist, 0, 0, _params); } } else { alglib.ap.assert(false, "SSA: integrity check in UpdateBasis() failed / dfgs34"); } } } // // Update recurrent relation // apserv.rvectorsetlengthatleast(ref s.forecasta, Math.Max(winw-1, 1), _params); degeneraterecurrence = false; if( winw>1 ) { // // Non-degenerate case // apserv.rvectorsetlengthatleast(ref s.tmp0, s.nbasis, _params); nu2 = 0.0; for(i=0; i<=s.nbasis-1; i++) { v = s.basist[i,winw-1]; s.tmp0[i] = v; nu2 = nu2+v*v; } if( (double)(nu2)<(double)(1-1000*math.machineepsilon) ) { ablas.rmatrixgemv(winw-1, s.nbasis, 1/(1-nu2), s.basist, 0, 0, 1, s.tmp0, 0, 0.0, s.forecasta, 0, _params); } else { degeneraterecurrence = true; } } else { degeneraterecurrence = true; } if( degeneraterecurrence ) { for(i=0; i<=Math.Max(winw-1, 1)-1; i++) { s.forecasta[i] = 0.0; } s.forecasta[Math.Max(winw-1, 1)-1] = 1.0; } // // Set validity flag // s.arebasisandsolvervalid = true; } /************************************************************************* This function performs analysis using current basis. It assumes and checks that validity flag AreBasisAndSolverValid is set. INPUT PARAMETERS: S - model Data - array which holds data in elements [I0,I1): * right bound is not included. * I1-I0>=WindowWidth (assertion is performed). Trend - preallocated output array, large enough Noise - preallocated output array, large enough Offs - offset in Trend/Noise where result is stored; I1-I0 elements are written starting at offset Offs. OUTPUT PARAMETERS: Trend, Noise - processing results -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ private static void analyzesequence(ssamodel s, double[] data, int i0, int i1, double[] trend, double[] noise, int offs, alglib.xparams _params) { int winw = 0; int nwindows = 0; int i = 0; int j = 0; int k = 0; int cnt = 0; int batchstart = 0; int batchlimit = 0; int batchsize = 0; alglib.ap.assert(s.arebasisandsolvervalid, "AnalyzeSequence: integrity check failed / d84sz0"); alglib.ap.assert(i1-i0>=s.windowwidth, "AnalyzeSequence: integrity check failed / d84sz1"); alglib.ap.assert(s.nbasis>=1, "AnalyzeSequence: integrity check failed / d84sz2"); nwindows = i1-i0-s.windowwidth+1; winw = s.windowwidth; batchlimit = Math.Max(nwindows, 1); if( s.memorylimit>0 ) { batchlimit = Math.Min(batchlimit, Math.Max(s.memorylimit/winw, 4*winw)); } // // Zero-initialize trend and counts // cnt = i1-i0; apserv.ivectorsetlengthatleast(ref s.aseqcounts, cnt, _params); for(i=0; i<=cnt-1; i++) { s.aseqcounts[i] = 0; trend[offs+i] = 0.0; } // // Reset temporaries if algorithm settings changed since last round // if( alglib.ap.cols(s.aseqtrajectory)!=winw ) { s.aseqtrajectory = new double[0, 0]; } if( alglib.ap.cols(s.aseqtbproduct)!=s.nbasis ) { s.aseqtbproduct = new double[0, 0]; } // // Perform batch processing // apserv.rmatrixsetlengthatleast(ref s.aseqtrajectory, batchlimit, winw, _params); apserv.rmatrixsetlengthatleast(ref s.aseqtbproduct, batchlimit, s.nbasis, _params); batchsize = 0; batchstart = offs; for(i=0; i<=nwindows-1; i++) { // // Enqueue next row of trajectory matrix // if( batchsize==0 ) { batchstart = i; } for(j=0; j<=winw-1; j++) { s.aseqtrajectory[batchsize,j] = data[i0+i+j]; } apserv.inc(ref batchsize, _params); // // Process batch // if( batchsize==batchlimit || i==nwindows-1 ) { // // Project onto basis // ablas.rmatrixgemm(batchsize, s.nbasis, winw, 1.0, s.aseqtrajectory, 0, 0, 0, s.basist, 0, 0, 1, 0.0, s.aseqtbproduct, 0, 0, _params); ablas.rmatrixgemm(batchsize, winw, s.nbasis, 1.0, s.aseqtbproduct, 0, 0, 0, s.basist, 0, 0, 0, 0.0, s.aseqtrajectory, 0, 0, _params); // // Hankelize // for(k=0; k<=batchsize-1; k++) { for(j=0; j<=winw-1; j++) { trend[offs+batchstart+k+j] = trend[offs+batchstart+k+j]+s.aseqtrajectory[k,j]; s.aseqcounts[batchstart+k+j] = s.aseqcounts[batchstart+k+j]+1; } } // // Reset batch size // batchsize = 0; } } for(i=0; i<=cnt-1; i++) { trend[offs+i] = trend[offs+i]/s.aseqcounts[i]; } // // Output noise // for(i=0; i<=cnt-1; i++) { noise[offs+i] = data[i0+i]-trend[offs+i]; } } /************************************************************************* This function performs averaged forecasting. It assumes that basis is already built, everything is valid and checked. See comments on similar public functions to find out more about averaged predictions. INPUT PARAMETERS: S - model Data - array which holds data in elements [I0,I1): * right bound is not included. * I1-I0>=WindowWidth (assertion is performed). M - number of sliding windows to combine, M>=1. If your dataset has less than M sliding windows, this parameter will be silently reduced. ForecastLen - number of ticks to predict, ForecastLen>=1 Trend - preallocated output array, large enough Offs - offset in Trend where result is stored; I1-I0 elements are written starting at offset Offs. OUTPUT PARAMETERS: Trend - array[ForecastLen], forecasted trend -- ALGLIB -- Copyright 30.10.2017 by Bochkanov Sergey *************************************************************************/ private static void forecastavgsequence(ssamodel s, double[] data, int i0, int i1, int m, int forecastlen, bool smooth, double[] trend, int offs, alglib.xparams _params) { int i = 0; int j = 0; int k = 0; int winw = 0; alglib.ap.assert(s.arebasisandsolvervalid, "ForecastAvgSequence: integrity check failed / d84sz0"); alglib.ap.assert(i1-i0-s.windowwidth+1>=m, "ForecastAvgSequence: integrity check failed / d84sz1"); alglib.ap.assert(s.nbasis>=1, "ForecastAvgSequence: integrity check failed / d84sz2"); alglib.ap.assert(s.windowwidth>=2, "ForecastAvgSequence: integrity check failed / 5tgdg5"); alglib.ap.assert(s.windowwidth>s.nbasis, "ForecastAvgSequence: integrity check failed / d5g56w"); winw = s.windowwidth; // // Prepare M synchronized predictions for the last known tick // (last one is an actual value of the trend, previous M-1 predictions // are predictions from differently positioned sliding windows). // apserv.rmatrixsetlengthatleast(ref s.fctrendm, m, winw, _params); apserv.rvectorsetlengthatleast(ref s.tmp0, Math.Max(m, s.nbasis), _params); apserv.rvectorsetlengthatleast(ref s.tmp1, winw, _params); for(k=0; k<=m-1; k++) { // // Perform prediction for rows [0,K-1] // ablas.rmatrixgemv(k, winw-1, 1.0, s.fctrendm, 0, 1, 0, s.forecasta, 0, 0.0, s.tmp0, 0, _params); for(i=0; i<=k-1; i++) { for(j=1; j<=winw-1; j++) { s.fctrendm[i,j-1] = s.fctrendm[i,j]; } s.fctrendm[i,winw-1] = s.tmp0[i]; } // // Perform trend extraction for row K, add it to dataset // if( smooth ) { ablas.rmatrixgemv(s.nbasis, winw, 1.0, s.basist, 0, 0, 0, data, i1-winw-(m-1-k), 0.0, s.tmp0, 0, _params); ablas.rmatrixgemv(s.windowwidth, s.nbasis, 1.0, s.basis, 0, 0, 0, s.tmp0, 0, 0.0, s.tmp1, 0, _params); for(j=0; j<=winw-1; j++) { s.fctrendm[k,j] = s.tmp1[j]; } } else { for(j=0; j<=winw-1; j++) { s.fctrendm[k,j] = data[i1-winw-(m-1-k)+j]; } } } // // Now we have M synchronized predictions of the sequence state at the last // know moment (last "prediction" is just a copy of the trend). Let's start // batch prediction! // for(k=0; k<=forecastlen-1; k++) { ablas.rmatrixgemv(m, winw-1, 1.0, s.fctrendm, 0, 1, 0, s.forecasta, 0, 0.0, s.tmp0, 0, _params); trend[offs+k] = 0.0; for(i=0; i<=m-1; i++) { for(j=1; j<=winw-1; j++) { s.fctrendm[i,j-1] = s.fctrendm[i,j]; } s.fctrendm[i,winw-1] = s.tmp0[i]; trend[offs+k] = trend[offs+k]+s.tmp0[i]; } trend[offs+k] = trend[offs+k]/m; } } /************************************************************************* This function extracts updates from real-time queue and applies them to the S.XXT matrix. XXT is premultiplied by Beta, which can be 0.0 for initial creation, 1.0 for subsequent updates, or even within (0,1) for some kind of updates with decay. INPUT PARAMETERS: S - model Beta - >=0, coefficient to premultiply XXT Cnt - 00, "SSA: RealTimeDequeue() integrity check failed / 43tdv"); alglib.ap.assert(math.isfinite(beta) && (double)(beta)>=(double)(0), "SSA: RealTimeDequeue() integrity check failed / 5gdg6"); alglib.ap.assert(cnt<=s.rtqueuecnt, "SSA: RealTimeDequeue() integrity check failed / 547yh"); alglib.ap.assert(alglib.ap.cols(s.xxt)>=s.windowwidth, "SSA: RealTimeDequeue() integrity check failed / 54bf4"); alglib.ap.assert(alglib.ap.rows(s.xxt)>=s.windowwidth, "SSA: RealTimeDequeue() integrity check failed / 9gdfn"); winw = s.windowwidth; // // Premultiply XXT by Beta // if( (double)(beta)!=(double)(0) ) { for(i=0; i<=winw-1; i++) { for(j=0; j<=winw-1; j++) { s.xxt[i,j] = s.xxt[i,j]*beta; } } } else { for(i=0; i<=winw-1; i++) { for(j=0; j<=winw-1; j++) { s.xxt[i,j] = 0; } } } // // Dequeue // updatexxtprepare(s, cnt, winw, s.memorylimit, _params); for(i=0; i<=cnt-1; i++) { updatexxtsend(s, s.sequencedata, s.rtqueue[s.rtqueuecnt-1], s.xxt, _params); apserv.dec(ref s.rtqueuecnt, _params); } updatexxtfinalize(s, s.xxt, _params); } /************************************************************************* This function prepares batch buffer for XXT update. The idea is that we send a stream of "XXT += u*u'" updates, and we want to package them into one big matrix update U*U', applied with SYRK() kernel, but U can consume too much memory, so we want to transparently divide it into few smaller chunks. This set of functions solves this problem: * UpdateXXTPrepare() prepares temporary buffers * UpdateXXTSend() sends next u to the buffer, possibly initiating next SYRK() * UpdateXXTFinalize() performs last SYRK() update INPUT PARAMETERS: S - model, only fields with UX prefix are used UpdateSize - number of updates WindowWidth - window width, >0 MemoryLimit - memory limit, non-positive value means no limit OUTPUT PARAMETERS: S - UX temporaries updated -- ALGLIB -- Copyright 20.12.2017 by Bochkanov Sergey *************************************************************************/ private static void updatexxtprepare(ssamodel s, int updatesize, int windowwidth, int memorylimit, alglib.xparams _params) { alglib.ap.assert(windowwidth>0, "UpdateXXTPrepare: WinW<=0"); s.uxbatchlimit = Math.Max(updatesize, 1); if( memorylimit>0 ) { s.uxbatchlimit = Math.Min(s.uxbatchlimit, Math.Max(memorylimit/windowwidth, 4*windowwidth)); } s.uxbatchwidth = windowwidth; s.uxbatchsize = 0; if( alglib.ap.cols(s.uxbatch)!=windowwidth ) { s.uxbatch = new double[0, 0]; } apserv.rmatrixsetlengthatleast(ref s.uxbatch, s.uxbatchlimit, windowwidth, _params); } /************************************************************************* This function sends update u*u' to the batch buffer. INPUT PARAMETERS: S - model, only fields with UX prefix are used U - WindowWidth-sized update, starts at I0 I0 - starting position for update OUTPUT PARAMETERS: S - UX temporaries updated XXT - array[WindowWidth,WindowWidth], in the middle of update. All intermediate updates are applied to the upper triangle. -- ALGLIB -- Copyright 20.12.2017 by Bochkanov Sergey *************************************************************************/ private static void updatexxtsend(ssamodel s, double[] u, int i0, double[,] xxt, alglib.xparams _params) { int i_ = 0; int i1_ = 0; alglib.ap.assert(i0+s.uxbatchwidth-1=0, "UpdateXXTSend: integrity check failure"); alglib.ap.assert(s.uxbatchsize<=s.uxbatchlimit, "UpdateXXTSend: integrity check failure"); alglib.ap.assert(s.uxbatchlimit>=1, "UpdateXXTSend: integrity check failure"); // // Send pending batch if full // if( s.uxbatchsize==s.uxbatchlimit ) { ablas.rmatrixsyrk(s.uxbatchwidth, s.uxbatchsize, 1.0, s.uxbatch, 0, 0, 2, 1.0, xxt, 0, 0, true, _params); s.uxbatchsize = 0; } // // Append update to batch // i1_ = (i0) - (0); for(i_=0; i_<=s.uxbatchwidth-1;i_++) { s.uxbatch[s.uxbatchsize,i_] = u[i_+i1_]; } apserv.inc(ref s.uxbatchsize, _params); } /************************************************************************* This function finalizes batch buffer. Call it after the last update. INPUT PARAMETERS: S - model, only fields with UX prefix are used OUTPUT PARAMETERS: S - UX temporaries updated XXT - array[WindowWidth,WindowWidth], updated with all previous updates, both triangles of the symmetric matrix are present. -- ALGLIB -- Copyright 20.12.2017 by Bochkanov Sergey *************************************************************************/ private static void updatexxtfinalize(ssamodel s, double[,] xxt, alglib.xparams _params) { alglib.ap.assert(s.uxbatchsize>=0, "UpdateXXTFinalize: integrity check failure"); alglib.ap.assert(s.uxbatchsize<=s.uxbatchlimit, "UpdateXXTFinalize: integrity check failure"); alglib.ap.assert(s.uxbatchlimit>=1, "UpdateXXTFinalize: integrity check failure"); if( s.uxbatchsize>0 ) { ablas.rmatrixsyrk(s.uxbatchwidth, s.uxbatchsize, 1.0, s.uxbatch, 0, 0, 2, 1.0, s.xxt, 0, 0, true, _params); s.uxbatchsize = 0; } ablas.rmatrixenforcesymmetricity(s.xxt, s.uxbatchwidth, true, _params); } } public class linreg { public class linearmodel : apobject { public double[] w; public linearmodel() { init(); } public override void init() { w = new double[0]; } public override alglib.apobject make_copy() { linearmodel _result = new linearmodel(); _result.w = (double[])w.Clone(); return _result; } }; /************************************************************************* LRReport structure contains additional information about linear model: * C - covariation matrix, array[0..NVars,0..NVars]. C[i,j] = Cov(A[i],A[j]) * RMSError - root mean square error on a training set * AvgError - average error on a training set * AvgRelError - average relative error on a training set (excluding observations with zero function value). * CVRMSError - leave-one-out cross-validation estimate of generalization error. Calculated using fast algorithm with O(NVars*NPoints) complexity. * CVAvgError - cross-validation estimate of average error * CVAvgRelError - cross-validation estimate of average relative error All other fields of the structure are intended for internal use and should not be used outside ALGLIB. *************************************************************************/ public class lrreport : apobject { public double[,] c; public double rmserror; public double avgerror; public double avgrelerror; public double cvrmserror; public double cvavgerror; public double cvavgrelerror; public int ncvdefects; public int[] cvdefects; public lrreport() { init(); } public override void init() { c = new double[0,0]; cvdefects = new int[0]; } public override alglib.apobject make_copy() { lrreport _result = new lrreport(); _result.c = (double[,])c.Clone(); _result.rmserror = rmserror; _result.avgerror = avgerror; _result.avgrelerror = avgrelerror; _result.cvrmserror = cvrmserror; _result.cvavgerror = cvavgerror; _result.cvavgrelerror = cvavgrelerror; _result.ncvdefects = ncvdefects; _result.cvdefects = (int[])cvdefects.Clone(); return _result; } }; public const int lrvnum = 5; /************************************************************************* Linear regression Subroutine builds model: Y = A(0)*X[0] + ... + A(N-1)*X[N-1] + A(N) and model found in ALGLIB format, covariation matrix, training set errors (rms, average, average relative) and leave-one-out cross-validation estimate of the generalization error. CV estimate calculated using fast algorithm with O(NPoints*NVars) complexity. When covariation matrix is calculated standard deviations of function values are assumed to be equal to RMS error on the training set. INPUT PARAMETERS: XY - training set, array [0..NPoints-1,0..NVars]: * NVars columns - independent variables * last column - dependent variable NPoints - training set size, NPoints>NVars+1 NVars - number of independent variables OUTPUT PARAMETERS: Info - return code: * -255, in case of unknown internal error * -4, if internal SVD subroutine haven't converged * -1, if incorrect parameters was passed (NPoints0. NPoints - training set size, NPoints>NVars+1 NVars - number of independent variables OUTPUT PARAMETERS: Info - return code: * -255, in case of unknown internal error * -4, if internal SVD subroutine haven't converged * -1, if incorrect parameters was passed (NPoints(double)(Math.Sqrt(variance)) ) { // // variation is relatively small, it is better to // bring mean value to 1 // c[j] = mean; } else { // // variation is large, it is better to bring variance to 1 // if( (double)(variance)==(double)(0) ) { variance = 1; } c[j] = Math.Sqrt(variance); } for(i=0; i<=npoints-1; i++) { xyi[i,j] = xyi[i,j]/c[j]; } } // // Internal processing // lrinternal(xyi, s, npoints, nvars+1, ref info, lm, ar, _params); if( info<0 ) { return; } // // Un-standartization // offs = (int)Math.Round(lm.w[3]); for(j=0; j<=nvars-1; j++) { // // J-th term is updated // lm.w[offs+j] = lm.w[offs+j]/c[j]; v = 1/c[j]; for(i_=0; i_<=nvars;i_++) { ar.c[j,i_] = v*ar.c[j,i_]; } for(i_=0; i_<=nvars;i_++) { ar.c[i_,j] = v*ar.c[i_,j]; } } } /************************************************************************* Like LRBuild but builds model Y = A(0)*X[0] + ... + A(N-1)*X[N-1] i.e. with zero constant term. -- ALGLIB -- Copyright 30.10.2008 by Bochkanov Sergey *************************************************************************/ public static void lrbuildz(double[,] xy, int npoints, int nvars, ref int info, linearmodel lm, lrreport ar, alglib.xparams _params) { double[] s = new double[0]; int i = 0; double sigma2 = 0; int i_ = 0; info = 0; if( npoints<=nvars+1 || nvars<1 ) { info = -1; return; } s = new double[npoints-1+1]; for(i=0; i<=npoints-1; i++) { s[i] = 1; } lrbuildzs(xy, s, npoints, nvars, ref info, lm, ar, _params); if( info<0 ) { return; } sigma2 = math.sqr(ar.rmserror)*npoints/(npoints-nvars-1); for(i=0; i<=nvars; i++) { for(i_=0; i_<=nvars;i_++) { ar.c[i,i_] = sigma2*ar.c[i,i_]; } } } /************************************************************************* Unpacks coefficients of linear model. INPUT PARAMETERS: LM - linear model in ALGLIB format OUTPUT PARAMETERS: V - coefficients, array[0..NVars] constant term (intercept) is stored in the V[NVars]. NVars - number of independent variables (one less than number of coefficients) -- ALGLIB -- Copyright 30.08.2008 by Bochkanov Sergey *************************************************************************/ public static void lrunpack(linearmodel lm, ref double[] v, ref int nvars, alglib.xparams _params) { int offs = 0; int i_ = 0; int i1_ = 0; v = new double[0]; nvars = 0; alglib.ap.assert((int)Math.Round(lm.w[1])==lrvnum, "LINREG: Incorrect LINREG version!"); nvars = (int)Math.Round(lm.w[2]); offs = (int)Math.Round(lm.w[3]); v = new double[nvars+1]; i1_ = (offs) - (0); for(i_=0; i_<=nvars;i_++) { v[i_] = lm.w[i_+i1_]; } } /************************************************************************* "Packs" coefficients and creates linear model in ALGLIB format (LRUnpack reversed). INPUT PARAMETERS: V - coefficients, array[0..NVars] NVars - number of independent variables OUTPUT PAREMETERS: LM - linear model. -- ALGLIB -- Copyright 30.08.2008 by Bochkanov Sergey *************************************************************************/ public static void lrpack(double[] v, int nvars, linearmodel lm, alglib.xparams _params) { int offs = 0; int i_ = 0; int i1_ = 0; lm.w = new double[4+nvars+1]; offs = 4; lm.w[0] = 4+nvars+1; lm.w[1] = lrvnum; lm.w[2] = nvars; lm.w[3] = offs; i1_ = (0) - (offs); for(i_=offs; i_<=offs+nvars;i_++) { lm.w[i_] = v[i_+i1_]; } } /************************************************************************* Procesing INPUT PARAMETERS: LM - linear model X - input vector, array[0..NVars-1]. Result: value of linear model regression estimate -- ALGLIB -- Copyright 03.09.2008 by Bochkanov Sergey *************************************************************************/ public static double lrprocess(linearmodel lm, double[] x, alglib.xparams _params) { double result = 0; double v = 0; int offs = 0; int nvars = 0; int i_ = 0; int i1_ = 0; alglib.ap.assert((int)Math.Round(lm.w[1])==lrvnum, "LINREG: Incorrect LINREG version!"); nvars = (int)Math.Round(lm.w[2]); offs = (int)Math.Round(lm.w[3]); i1_ = (offs)-(0); v = 0.0; for(i_=0; i_<=nvars-1;i_++) { v += x[i_]*lm.w[i_+i1_]; } result = v+lm.w[offs+nvars]; return result; } /************************************************************************* RMS error on the test set INPUT PARAMETERS: LM - linear model XY - test set NPoints - test set size RESULT: root mean square error. -- ALGLIB -- Copyright 30.08.2008 by Bochkanov Sergey *************************************************************************/ public static double lrrmserror(linearmodel lm, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; int i = 0; double v = 0; int offs = 0; int nvars = 0; int i_ = 0; int i1_ = 0; alglib.ap.assert((int)Math.Round(lm.w[1])==lrvnum, "LINREG: Incorrect LINREG version!"); nvars = (int)Math.Round(lm.w[2]); offs = (int)Math.Round(lm.w[3]); result = 0; for(i=0; i<=npoints-1; i++) { i1_ = (offs)-(0); v = 0.0; for(i_=0; i_<=nvars-1;i_++) { v += xy[i,i_]*lm.w[i_+i1_]; } v = v+lm.w[offs+nvars]; result = result+math.sqr(v-xy[i,nvars]); } result = Math.Sqrt(result/npoints); return result; } /************************************************************************* Average error on the test set INPUT PARAMETERS: LM - linear model XY - test set NPoints - test set size RESULT: average error. -- ALGLIB -- Copyright 30.08.2008 by Bochkanov Sergey *************************************************************************/ public static double lravgerror(linearmodel lm, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; int i = 0; double v = 0; int offs = 0; int nvars = 0; int i_ = 0; int i1_ = 0; alglib.ap.assert((int)Math.Round(lm.w[1])==lrvnum, "LINREG: Incorrect LINREG version!"); nvars = (int)Math.Round(lm.w[2]); offs = (int)Math.Round(lm.w[3]); result = 0; for(i=0; i<=npoints-1; i++) { i1_ = (offs)-(0); v = 0.0; for(i_=0; i_<=nvars-1;i_++) { v += xy[i,i_]*lm.w[i_+i1_]; } v = v+lm.w[offs+nvars]; result = result+Math.Abs(v-xy[i,nvars]); } result = result/npoints; return result; } /************************************************************************* RMS error on the test set INPUT PARAMETERS: LM - linear model XY - test set NPoints - test set size RESULT: average relative error. -- ALGLIB -- Copyright 30.08.2008 by Bochkanov Sergey *************************************************************************/ public static double lravgrelerror(linearmodel lm, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; int i = 0; int k = 0; double v = 0; int offs = 0; int nvars = 0; int i_ = 0; int i1_ = 0; alglib.ap.assert((int)Math.Round(lm.w[1])==lrvnum, "LINREG: Incorrect LINREG version!"); nvars = (int)Math.Round(lm.w[2]); offs = (int)Math.Round(lm.w[3]); result = 0; k = 0; for(i=0; i<=npoints-1; i++) { if( (double)(xy[i,nvars])!=(double)(0) ) { i1_ = (offs)-(0); v = 0.0; for(i_=0; i_<=nvars-1;i_++) { v += xy[i,i_]*lm.w[i_+i1_]; } v = v+lm.w[offs+nvars]; result = result+Math.Abs((v-xy[i,nvars])/xy[i,nvars]); k = k+1; } } if( k!=0 ) { result = result/k; } return result; } /************************************************************************* Copying of LinearModel strucure INPUT PARAMETERS: LM1 - original OUTPUT PARAMETERS: LM2 - copy -- ALGLIB -- Copyright 15.03.2009 by Bochkanov Sergey *************************************************************************/ public static void lrcopy(linearmodel lm1, linearmodel lm2, alglib.xparams _params) { int k = 0; int i_ = 0; k = (int)Math.Round(lm1.w[0]); lm2.w = new double[k-1+1]; for(i_=0; i_<=k-1;i_++) { lm2.w[i_] = lm1.w[i_]; } } public static void lrlines(double[,] xy, double[] s, int n, ref int info, ref double a, ref double b, ref double vara, ref double varb, ref double covab, ref double corrab, ref double p, alglib.xparams _params) { int i = 0; double ss = 0; double sx = 0; double sxx = 0; double sy = 0; double stt = 0; double e1 = 0; double e2 = 0; double t = 0; double chi2 = 0; info = 0; a = 0; b = 0; vara = 0; varb = 0; covab = 0; corrab = 0; p = 0; if( n<2 ) { info = -1; return; } for(i=0; i<=n-1; i++) { if( (double)(s[i])<=(double)(0) ) { info = -2; return; } } info = 1; // // Calculate S, SX, SY, SXX // ss = 0; sx = 0; sy = 0; sxx = 0; for(i=0; i<=n-1; i++) { t = math.sqr(s[i]); ss = ss+1/t; sx = sx+xy[i,0]/t; sy = sy+xy[i,1]/t; sxx = sxx+math.sqr(xy[i,0])/t; } // // Test for condition number // t = Math.Sqrt(4*math.sqr(sx)+math.sqr(ss-sxx)); e1 = 0.5*(ss+sxx+t); e2 = 0.5*(ss+sxx-t); if( (double)(Math.Min(e1, e2))<=(double)(1000*math.machineepsilon*Math.Max(e1, e2)) ) { info = -3; return; } // // Calculate A, B // a = 0; b = 0; stt = 0; for(i=0; i<=n-1; i++) { t = (xy[i,0]-sx/ss)/s[i]; b = b+t*xy[i,1]/s[i]; stt = stt+math.sqr(t); } b = b/stt; a = (sy-sx*b)/ss; // // Calculate goodness-of-fit // if( n>2 ) { chi2 = 0; for(i=0; i<=n-1; i++) { chi2 = chi2+math.sqr((xy[i,1]-a-b*xy[i,0])/s[i]); } p = igammaf.incompletegammac((double)(n-2)/(double)2, chi2/2, _params); } else { p = 1; } // // Calculate other parameters // vara = (1+math.sqr(sx)/(ss*stt))/ss; varb = 1/stt; covab = -(sx/(ss*stt)); corrab = covab/Math.Sqrt(vara*varb); } public static void lrline(double[,] xy, int n, ref int info, ref double a, ref double b, alglib.xparams _params) { double[] s = new double[0]; int i = 0; double vara = 0; double varb = 0; double covab = 0; double corrab = 0; double p = 0; info = 0; a = 0; b = 0; if( n<2 ) { info = -1; return; } s = new double[n-1+1]; for(i=0; i<=n-1; i++) { s[i] = 1; } lrlines(xy, s, n, ref info, ref a, ref b, ref vara, ref varb, ref covab, ref corrab, ref p, _params); } /************************************************************************* Internal linear regression subroutine *************************************************************************/ private static void lrinternal(double[,] xy, double[] s, int npoints, int nvars, ref int info, linearmodel lm, lrreport ar, alglib.xparams _params) { double[,] a = new double[0,0]; double[,] u = new double[0,0]; double[,] vt = new double[0,0]; double[,] vm = new double[0,0]; double[,] xym = new double[0,0]; double[] b = new double[0]; double[] sv = new double[0]; double[] t = new double[0]; double[] svi = new double[0]; double[] work = new double[0]; int i = 0; int j = 0; int k = 0; int ncv = 0; int na = 0; int nacv = 0; double r = 0; double p = 0; double epstol = 0; lrreport ar2 = new lrreport(); int offs = 0; linearmodel tlm = new linearmodel(); int i_ = 0; int i1_ = 0; info = 0; epstol = 1000; // // Check for errors in data // if( npoints=1; k--) { if( (double)(sv[k-1])>(double)(epstol*math.machineepsilon*sv[0]) ) { // // Reduce // xym = new double[npoints-1+1, k+1]; for(i=0; i<=npoints-1; i++) { for(j=0; j<=k-1; j++) { r = 0.0; for(i_=0; i_<=nvars-1;i_++) { r += xy[i,i_]*vt[j,i_]; } xym[i,j] = r; } xym[i,k] = xy[i,nvars]; } // // Solve // lrinternal(xym, s, npoints, k, ref info, tlm, ar2, _params); if( info!=1 ) { return; } // // Convert back to un-reduced format // for(j=0; j<=nvars-1; j++) { lm.w[offs+j] = 0; } for(j=0; j<=k-1; j++) { r = tlm.w[offs+j]; i1_ = (0) - (offs); for(i_=offs; i_<=offs+nvars-1;i_++) { lm.w[i_] = lm.w[i_] + r*vt[j,i_+i1_]; } } ar.rmserror = ar2.rmserror; ar.avgerror = ar2.avgerror; ar.avgrelerror = ar2.avgrelerror; ar.cvrmserror = ar2.cvrmserror; ar.cvavgerror = ar2.cvavgerror; ar.cvavgrelerror = ar2.cvavgrelerror; ar.ncvdefects = ar2.ncvdefects; ar.cvdefects = new int[nvars-1+1]; for(j=0; j<=ar.ncvdefects-1; j++) { ar.cvdefects[j] = ar2.cvdefects[j]; } for(j=ar.ncvdefects; j<=nvars-1; j++) { ar.cvdefects[j] = -1; } ar.c = new double[nvars-1+1, nvars-1+1]; work = new double[nvars+1]; blas.matrixmatrixmultiply(ar2.c, 0, k-1, 0, k-1, false, vt, 0, k-1, 0, nvars-1, false, 1.0, ref vm, 0, k-1, 0, nvars-1, 0.0, ref work, _params); blas.matrixmatrixmultiply(vt, 0, k-1, 0, nvars-1, true, vm, 0, k-1, 0, nvars-1, false, 1.0, ref ar.c, 0, nvars-1, 0, nvars-1, 0.0, ref work, _params); return; } } info = -255; return; } for(i=0; i<=nvars-1; i++) { if( (double)(sv[i])>(double)(epstol*math.machineepsilon*sv[0]) ) { svi[i] = 1/sv[i]; } else { svi[i] = 0; } } for(i=0; i<=nvars-1; i++) { t[i] = 0; } for(i=0; i<=npoints-1; i++) { r = b[i]; for(i_=0; i_<=nvars-1;i_++) { t[i_] = t[i_] + r*u[i,i_]; } } for(i=0; i<=nvars-1; i++) { lm.w[offs+i] = 0; } for(i=0; i<=nvars-1; i++) { r = t[i]*svi[i]; i1_ = (0) - (offs); for(i_=offs; i_<=offs+nvars-1;i_++) { lm.w[i_] = lm.w[i_] + r*vt[i,i_+i1_]; } } for(j=0; j<=nvars-1; j++) { r = svi[j]; for(i_=0; i_<=nvars-1;i_++) { vm[i_,j] = r*vt[j,i_]; } } for(i=0; i<=nvars-1; i++) { for(j=i; j<=nvars-1; j++) { r = 0.0; for(i_=0; i_<=nvars-1;i_++) { r += vm[i,i_]*vm[j,i_]; } ar.c[i,j] = r; ar.c[j,i] = r; } } // // Leave-1-out cross-validation error. // // NOTATIONS: // A design matrix // A*x = b original linear least squares task // U*S*V' SVD of A // ai i-th row of the A // bi i-th element of the b // xf solution of the original LLS task // // Cross-validation error of i-th element from a sample is // calculated using following formula: // // ERRi = ai*xf - (ai*xf-bi*(ui*ui'))/(1-ui*ui') (1) // // This formula can be derived from normal equations of the // original task // // (A'*A)x = A'*b (2) // // by applying modification (zeroing out i-th row of A) to (2): // // (A-ai)'*(A-ai) = (A-ai)'*b // // and using Sherman-Morrison formula for updating matrix inverse // // NOTE 1: b is not zeroed out since it is much simpler and // does not influence final result. // // NOTE 2: some design matrices A have such ui that 1-ui*ui'=0. // Formula (1) can't be applied for such cases and they are skipped // from CV calculation (which distorts resulting CV estimate). // But from the properties of U we can conclude that there can // be no more than NVars such vectors. Usually // NVars << NPoints, so in a normal case it only slightly // influences result. // ncv = 0; na = 0; nacv = 0; ar.rmserror = 0; ar.avgerror = 0; ar.avgrelerror = 0; ar.cvrmserror = 0; ar.cvavgerror = 0; ar.cvavgrelerror = 0; ar.ncvdefects = 0; ar.cvdefects = new int[nvars-1+1]; for(i=0; i<=nvars-1; i++) { ar.cvdefects[i] = -1; } for(i=0; i<=npoints-1; i++) { // // Error on a training set // i1_ = (offs)-(0); r = 0.0; for(i_=0; i_<=nvars-1;i_++) { r += xy[i,i_]*lm.w[i_+i1_]; } ar.rmserror = ar.rmserror+math.sqr(r-xy[i,nvars]); ar.avgerror = ar.avgerror+Math.Abs(r-xy[i,nvars]); if( (double)(xy[i,nvars])!=(double)(0) ) { ar.avgrelerror = ar.avgrelerror+Math.Abs((r-xy[i,nvars])/xy[i,nvars]); na = na+1; } // // Error using fast leave-one-out cross-validation // p = 0.0; for(i_=0; i_<=nvars-1;i_++) { p += u[i,i_]*u[i,i_]; } if( (double)(p)>(double)(1-epstol*math.machineepsilon) ) { ar.cvdefects[ar.ncvdefects] = i; ar.ncvdefects = ar.ncvdefects+1; continue; } r = s[i]*(r/s[i]-b[i]*p)/(1-p); ar.cvrmserror = ar.cvrmserror+math.sqr(r-xy[i,nvars]); ar.cvavgerror = ar.cvavgerror+Math.Abs(r-xy[i,nvars]); if( (double)(xy[i,nvars])!=(double)(0) ) { ar.cvavgrelerror = ar.cvavgrelerror+Math.Abs((r-xy[i,nvars])/xy[i,nvars]); nacv = nacv+1; } ncv = ncv+1; } if( ncv==0 ) { // // Something strange: ALL ui are degenerate. // Unexpected... // info = -255; return; } ar.rmserror = Math.Sqrt(ar.rmserror/npoints); ar.avgerror = ar.avgerror/npoints; if( na!=0 ) { ar.avgrelerror = ar.avgrelerror/na; } ar.cvrmserror = Math.Sqrt(ar.cvrmserror/ncv); ar.cvavgerror = ar.cvavgerror/ncv; if( nacv!=0 ) { ar.cvavgrelerror = ar.cvavgrelerror/nacv; } } } public class filters { /************************************************************************* Filters: simple moving averages (unsymmetric). This filter replaces array by results of SMA(K) filter. SMA(K) is defined as filter which averages at most K previous points (previous - not points AROUND central point) - or less, in case of the first K-1 points. INPUT PARAMETERS: X - array[N], array to process. It can be larger than N, in this case only first N points are processed. N - points count, N>=0 K - K>=1 (K can be larger than N , such cases will be correctly handled). Window width. K=1 corresponds to identity transformation (nothing changes). OUTPUT PARAMETERS: X - array, whose first N elements were processed with SMA(K) NOTE 1: this function uses efficient in-place algorithm which does not allocate temporary arrays. NOTE 2: this algorithm makes only one pass through array and uses running sum to speed-up calculation of the averages. Additional measures are taken to ensure that running sum on a long sequence of zero elements will be correctly reset to zero even in the presence of round-off error. NOTE 3: this is unsymmetric version of the algorithm, which does NOT averages points after the current one. Only X[i], X[i-1], ... are used when calculating new value of X[i]. We should also note that this algorithm uses BOTH previous points and current one, i.e. new value of X[i] depends on BOTH previous point and X[i] itself. -- ALGLIB -- Copyright 25.10.2011 by Bochkanov Sergey *************************************************************************/ public static void filtersma(ref double[] x, int n, int k, alglib.xparams _params) { int i = 0; double runningsum = 0; double termsinsum = 0; int zeroprefix = 0; double v = 0; alglib.ap.assert(n>=0, "FilterSMA: N<0"); alglib.ap.assert(alglib.ap.len(x)>=n, "FilterSMA: Length(X)=1, "FilterSMA: K<1"); // // Quick exit, if necessary // if( n<=1 || k==1 ) { return; } // // Prepare variables (see below for explanation) // runningsum = 0.0; termsinsum = 0; for(i=Math.Max(n-k, 0); i<=n-1; i++) { runningsum = runningsum+x[i]; termsinsum = termsinsum+1; } i = Math.Max(n-k, 0); zeroprefix = 0; while( i<=n-1 && (double)(x[i])==(double)(0) ) { zeroprefix = zeroprefix+1; i = i+1; } // // General case: we assume that N>1 and K>1 // // Make one pass through all elements. At the beginning of // the iteration we have: // * I element being processed // * RunningSum current value of the running sum // (including I-th element) // * TermsInSum number of terms in sum, 0<=TermsInSum<=K // * ZeroPrefix length of the sequence of zero elements // which starts at X[I-K+1] and continues towards X[I]. // Equal to zero in case X[I-K+1] is non-zero. // This value is used to make RunningSum exactly zero // when it follows from the problem properties. // for(i=n-1; i>=0; i--) { // // Store new value of X[i], save old value in V // v = x[i]; x[i] = runningsum/termsinsum; // // Update RunningSum and TermsInSum // if( i-k>=0 ) { runningsum = runningsum-v+x[i-k]; } else { runningsum = runningsum-v; termsinsum = termsinsum-1; } // // Update ZeroPrefix. // In case we have ZeroPrefix=TermsInSum, // RunningSum is reset to zero. // if( i-k>=0 ) { if( (double)(x[i-k])!=(double)(0) ) { zeroprefix = 0; } else { zeroprefix = Math.Min(zeroprefix+1, k); } } else { zeroprefix = Math.Min(zeroprefix, i+1); } if( (double)(zeroprefix)==(double)(termsinsum) ) { runningsum = 0; } } } /************************************************************************* Filters: exponential moving averages. This filter replaces array by results of EMA(alpha) filter. EMA(alpha) is defined as filter which replaces X[] by S[]: S[0] = X[0] S[t] = alpha*X[t] + (1-alpha)*S[t-1] INPUT PARAMETERS: X - array[N], array to process. It can be larger than N, in this case only first N points are processed. N - points count, N>=0 alpha - 0=0, "FilterEMA: N<0"); alglib.ap.assert(alglib.ap.len(x)>=n, "FilterEMA: Length(X)(double)(0), "FilterEMA: Alpha<=0"); alglib.ap.assert((double)(alpha)<=(double)(1), "FilterEMA: Alpha>1"); // // Quick exit, if necessary // if( n<=1 || (double)(alpha)==(double)(1) ) { return; } // // Process // for(i=1; i<=n-1; i++) { x[i] = alpha*x[i]+(1-alpha)*x[i-1]; } } /************************************************************************* Filters: linear regression moving averages. This filter replaces array by results of LRMA(K) filter. LRMA(K) is defined as filter which, for each data point, builds linear regression model using K prevous points (point itself is included in these K points) and calculates value of this linear model at the point in question. INPUT PARAMETERS: X - array[N], array to process. It can be larger than N, in this case only first N points are processed. N - points count, N>=0 K - K>=1 (K can be larger than N , such cases will be correctly handled). Window width. K=1 corresponds to identity transformation (nothing changes). OUTPUT PARAMETERS: X - array, whose first N elements were processed with SMA(K) NOTE 1: this function uses efficient in-place algorithm which does not allocate temporary arrays. NOTE 2: this algorithm makes only one pass through array and uses running sum to speed-up calculation of the averages. Additional measures are taken to ensure that running sum on a long sequence of zero elements will be correctly reset to zero even in the presence of round-off error. NOTE 3: this is unsymmetric version of the algorithm, which does NOT averages points after the current one. Only X[i], X[i-1], ... are used when calculating new value of X[i]. We should also note that this algorithm uses BOTH previous points and current one, i.e. new value of X[i] depends on BOTH previous point and X[i] itself. -- ALGLIB -- Copyright 25.10.2011 by Bochkanov Sergey *************************************************************************/ public static void filterlrma(ref double[] x, int n, int k, alglib.xparams _params) { int i = 0; int m = 0; double[,] xy = new double[0,0]; double[] s = new double[0]; int info = 0; double a = 0; double b = 0; double vara = 0; double varb = 0; double covab = 0; double corrab = 0; double p = 0; int i_ = 0; int i1_ = 0; alglib.ap.assert(n>=0, "FilterLRMA: N<0"); alglib.ap.assert(alglib.ap.len(x)>=n, "FilterLRMA: Length(X)=1, "FilterLRMA: K<1"); // // Quick exit, if necessary: // * either N is equal to 1 (nothing to average) // * or K is 1 (only point itself is used) or 2 (model is too simple, // we will always get identity transformation) // if( n<=1 || k<=2 ) { return; } // // General case: K>2, N>1. // We do not process points with I<2 because first two points (I=0 and I=1) will be // left unmodified by LRMA filter in any case. // xy = new double[k, 2]; s = new double[k]; for(i=0; i<=k-1; i++) { xy[i,0] = i; s[i] = 1.0; } for(i=n-1; i>=2; i--) { m = Math.Min(i+1, k); i1_ = (i-m+1) - (0); for(i_=0; i_<=m-1;i_++) { xy[i_,1] = x[i_+i1_]; } linreg.lrlines(xy, s, m, ref info, ref a, ref b, ref vara, ref varb, ref covab, ref corrab, ref p, _params); alglib.ap.assert(info==1, "FilterLRMA: internal error"); x[i] = a+b*(m-1); } } } public class logit { public class logitmodel : apobject { public double[] w; public logitmodel() { init(); } public override void init() { w = new double[0]; } public override alglib.apobject make_copy() { logitmodel _result = new logitmodel(); _result.w = (double[])w.Clone(); return _result; } }; public class logitmcstate : apobject { public bool brackt; public bool stage1; public int infoc; public double dg; public double dgm; public double dginit; public double dgtest; public double dgx; public double dgxm; public double dgy; public double dgym; public double finit; public double ftest1; public double fm; public double fx; public double fxm; public double fy; public double fym; public double stx; public double sty; public double stmin; public double stmax; public double width; public double width1; public double xtrapf; public logitmcstate() { init(); } public override void init() { } public override alglib.apobject make_copy() { logitmcstate _result = new logitmcstate(); _result.brackt = brackt; _result.stage1 = stage1; _result.infoc = infoc; _result.dg = dg; _result.dgm = dgm; _result.dginit = dginit; _result.dgtest = dgtest; _result.dgx = dgx; _result.dgxm = dgxm; _result.dgy = dgy; _result.dgym = dgym; _result.finit = finit; _result.ftest1 = ftest1; _result.fm = fm; _result.fx = fx; _result.fxm = fxm; _result.fy = fy; _result.fym = fym; _result.stx = stx; _result.sty = sty; _result.stmin = stmin; _result.stmax = stmax; _result.width = width; _result.width1 = width1; _result.xtrapf = xtrapf; return _result; } }; /************************************************************************* MNLReport structure contains information about training process: * NGrad - number of gradient calculations * NHess - number of Hessian calculations *************************************************************************/ public class mnlreport : apobject { public int ngrad; public int nhess; public mnlreport() { init(); } public override void init() { } public override alglib.apobject make_copy() { mnlreport _result = new mnlreport(); _result.ngrad = ngrad; _result.nhess = nhess; return _result; } }; public const double xtol = 100*math.machineepsilon; public const double ftol = 0.0001; public const double gtol = 0.3; public const int maxfev = 20; public const double stpmin = 1.0E-2; public const double stpmax = 1.0E5; public const int logitvnum = 6; /************************************************************************* This subroutine trains logit model. INPUT PARAMETERS: XY - training set, array[0..NPoints-1,0..NVars] First NVars columns store values of independent variables, next column stores number of class (from 0 to NClasses-1) which dataset element belongs to. Fractional values are rounded to nearest integer. NPoints - training set size, NPoints>=1 NVars - number of independent variables, NVars>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: Info - return code: * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints=nclasses ) { info = -2; return; } } info = 1; // // Initialize data // rep.ngrad = 0; rep.nhess = 0; // // Allocate array // offs = 5; ssize = 5+(nvars+1)*(nclasses-1)+nclasses; lm.w = new double[ssize-1+1]; lm.w[0] = ssize; lm.w[1] = logitvnum; lm.w[2] = nvars; lm.w[3] = nclasses; lm.w[4] = offs; // // Degenerate case: all outputs are equal // allsame = true; for(i=1; i<=npoints-1; i++) { if( (int)Math.Round(xy[i,nvars])!=(int)Math.Round(xy[i-1,nvars]) ) { allsame = false; } } if( allsame ) { for(i=0; i<=(nvars+1)*(nclasses-1)-1; i++) { lm.w[offs+i] = 0; } v = -(2*Math.Log(math.minrealnumber)); k = (int)Math.Round(xy[0,nvars]); if( k==nclasses-1 ) { for(i=0; i<=nclasses-2; i++) { lm.w[offs+i*(nvars+1)+nvars] = -v; } } else { for(i=0; i<=nclasses-2; i++) { if( i==k ) { lm.w[offs+i*(nvars+1)+nvars] = v; } else { lm.w[offs+i*(nvars+1)+nvars] = 0; } } } return; } // // General case. // Prepare task and network. Allocate space. // mlpbase.mlpcreatec0(nvars, nclasses, network, _params); mlpbase.mlpinitpreprocessor(network, xy, npoints, _params); mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount, _params); for(i=0; i<=wcount-1; i++) { network.weights[i] = (2*math.randomreal()-1)/nvars; } g = new double[wcount-1+1]; h = new double[wcount-1+1, wcount-1+1]; wbase = new double[wcount-1+1]; wdir = new double[wcount-1+1]; work = new double[wcount-1+1]; // // First stage: optimize in gradient direction. // for(k=0; k<=wcount/3+10; k++) { // // Calculate gradient in starting point // mlpbase.mlpgradnbatch(network, xy, npoints, ref e, ref g, _params); v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } e = e+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { g[i_] = g[i_] + decay*network.weights[i_]; } rep.ngrad = rep.ngrad+1; // // Setup optimization scheme // for(i_=0; i_<=wcount-1;i_++) { wdir[i_] = -g[i_]; } v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += wdir[i_]*wdir[i_]; } wstep = Math.Sqrt(v); v = 1/Math.Sqrt(v); for(i_=0; i_<=wcount-1;i_++) { wdir[i_] = v*wdir[i_]; } mcstage = 0; mnlmcsrch(wcount, ref network.weights, ref e, ref g, wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, mcstate, ref mcstage, _params); while( mcstage!=0 ) { mlpbase.mlpgradnbatch(network, xy, npoints, ref e, ref g, _params); v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } e = e+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { g[i_] = g[i_] + decay*network.weights[i_]; } rep.ngrad = rep.ngrad+1; mnlmcsrch(wcount, ref network.weights, ref e, ref g, wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, mcstate, ref mcstage, _params); } } // // Second stage: use Hessian when we are close to the minimum // while( true ) { // // Calculate and update E/G/H // mlpbase.mlphessiannbatch(network, xy, npoints, ref e, ref g, ref h, _params); v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } e = e+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { g[i_] = g[i_] + decay*network.weights[i_]; } for(k=0; k<=wcount-1; k++) { h[k,k] = h[k,k]+decay; } rep.nhess = rep.nhess+1; // // Select step direction // NOTE: it is important to use lower-triangle Cholesky // factorization since it is much faster than higher-triangle version. // spd = trfac.spdmatrixcholesky(ref h, wcount, false, _params); directdensesolvers.spdmatrixcholeskysolve(h, wcount, false, g, ref solverinfo, solverrep, ref wdir, _params); spd = solverinfo>0; if( spd ) { // // H is positive definite. // Step in Newton direction. // for(i_=0; i_<=wcount-1;i_++) { wdir[i_] = -1*wdir[i_]; } spd = true; } else { // // H is indefinite. // Step in gradient direction. // for(i_=0; i_<=wcount-1;i_++) { wdir[i_] = -g[i_]; } spd = false; } // // Optimize in WDir direction // v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += wdir[i_]*wdir[i_]; } wstep = Math.Sqrt(v); v = 1/Math.Sqrt(v); for(i_=0; i_<=wcount-1;i_++) { wdir[i_] = v*wdir[i_]; } mcstage = 0; mnlmcsrch(wcount, ref network.weights, ref e, ref g, wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, mcstate, ref mcstage, _params); while( mcstage!=0 ) { mlpbase.mlpgradnbatch(network, xy, npoints, ref e, ref g, _params); v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } e = e+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { g[i_] = g[i_] + decay*network.weights[i_]; } rep.ngrad = rep.ngrad+1; mnlmcsrch(wcount, ref network.weights, ref e, ref g, wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, mcstate, ref mcstage, _params); } if( spd && ((mcinfo==2 || mcinfo==4) || mcinfo==6) ) { break; } } // // Convert from NN format to MNL format // i1_ = (0) - (offs); for(i_=offs; i_<=offs+wcount-1;i_++) { lm.w[i_] = network.weights[i_+i1_]; } for(k=0; k<=nvars-1; k++) { for(i=0; i<=nclasses-2; i++) { s = network.columnsigmas[k]; if( (double)(s)==(double)(0) ) { s = 1; } j = offs+(nvars+1)*i; v = lm.w[j+k]; lm.w[j+k] = v/s; lm.w[j+nvars] = lm.w[j+nvars]+v*network.columnmeans[k]/s; } } for(k=0; k<=nclasses-2; k++) { lm.w[offs+(nvars+1)*k+nvars] = -lm.w[offs+(nvars+1)*k+nvars]; } } /************************************************************************* Procesing INPUT PARAMETERS: LM - logit model, passed by non-constant reference (some fields of structure are used as temporaries when calculating model output). X - input vector, array[0..NVars-1]. Y - (possibly) preallocated buffer; if size of Y is less than NClasses, it will be reallocated.If it is large enough, it is NOT reallocated, so we can save some time on reallocation. OUTPUT PARAMETERS: Y - result, array[0..NClasses-1] Vector of posterior probabilities for classification task. -- ALGLIB -- Copyright 10.09.2008 by Bochkanov Sergey *************************************************************************/ public static void mnlprocess(logitmodel lm, double[] x, ref double[] y, alglib.xparams _params) { int nvars = 0; int nclasses = 0; int offs = 0; int i = 0; int i1 = 0; double s = 0; alglib.ap.assert((double)(lm.w[1])==(double)(logitvnum), "MNLProcess: unexpected model version"); nvars = (int)Math.Round(lm.w[2]); nclasses = (int)Math.Round(lm.w[3]); offs = (int)Math.Round(lm.w[4]); mnliexp(ref lm.w, x, _params); s = 0; i1 = offs+(nvars+1)*(nclasses-1); for(i=i1; i<=i1+nclasses-1; i++) { s = s+lm.w[i]; } if( alglib.ap.len(y)=0 && (int)Math.Round(xy[i,nvars])(double)(0) ) { result = result-Math.Log(worky[(int)Math.Round(xy[i,nvars])]); } else { result = result-Math.Log(math.minrealnumber); } } result = result/(npoints*Math.Log(2)); return result; } /************************************************************************* Relative classification error on the test set INPUT PARAMETERS: LM - logit model XY - test set NPoints - test set size RESULT: percent of incorrectly classified cases. -- ALGLIB -- Copyright 10.09.2008 by Bochkanov Sergey *************************************************************************/ public static double mnlrelclserror(logitmodel lm, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; result = (double)mnlclserror(lm, xy, npoints, _params)/(double)npoints; return result; } /************************************************************************* RMS error on the test set INPUT PARAMETERS: LM - logit model XY - test set NPoints - test set size RESULT: root mean square error (error when estimating posterior probabilities). -- ALGLIB -- Copyright 30.08.2008 by Bochkanov Sergey *************************************************************************/ public static double mnlrmserror(logitmodel lm, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; double relcls = 0; double avgce = 0; double rms = 0; double avg = 0; double avgrel = 0; alglib.ap.assert((int)Math.Round(lm.w[1])==logitvnum, "MNLRMSError: Incorrect MNL version!"); mnlallerrors(lm, xy, npoints, ref relcls, ref avgce, ref rms, ref avg, ref avgrel, _params); result = rms; return result; } /************************************************************************* Average error on the test set INPUT PARAMETERS: LM - logit model XY - test set NPoints - test set size RESULT: average error (error when estimating posterior probabilities). -- ALGLIB -- Copyright 30.08.2008 by Bochkanov Sergey *************************************************************************/ public static double mnlavgerror(logitmodel lm, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; double relcls = 0; double avgce = 0; double rms = 0; double avg = 0; double avgrel = 0; alglib.ap.assert((int)Math.Round(lm.w[1])==logitvnum, "MNLRMSError: Incorrect MNL version!"); mnlallerrors(lm, xy, npoints, ref relcls, ref avgce, ref rms, ref avg, ref avgrel, _params); result = avg; return result; } /************************************************************************* Average relative error on the test set INPUT PARAMETERS: LM - logit model XY - test set NPoints - test set size RESULT: average relative error (error when estimating posterior probabilities). -- ALGLIB -- Copyright 30.08.2008 by Bochkanov Sergey *************************************************************************/ public static double mnlavgrelerror(logitmodel lm, double[,] xy, int ssize, alglib.xparams _params) { double result = 0; double relcls = 0; double avgce = 0; double rms = 0; double avg = 0; double avgrel = 0; alglib.ap.assert((int)Math.Round(lm.w[1])==logitvnum, "MNLRMSError: Incorrect MNL version!"); mnlallerrors(lm, xy, ssize, ref relcls, ref avgce, ref rms, ref avg, ref avgrel, _params); result = avgrel; return result; } /************************************************************************* Classification error on test set = MNLRelClsError*NPoints -- ALGLIB -- Copyright 10.09.2008 by Bochkanov Sergey *************************************************************************/ public static int mnlclserror(logitmodel lm, double[,] xy, int npoints, alglib.xparams _params) { int result = 0; int nvars = 0; int nclasses = 0; int i = 0; int j = 0; double[] workx = new double[0]; double[] worky = new double[0]; int nmax = 0; int i_ = 0; alglib.ap.assert((double)(lm.w[1])==(double)(logitvnum), "MNLClsError: unexpected model version"); nvars = (int)Math.Round(lm.w[2]); nclasses = (int)Math.Round(lm.w[3]); workx = new double[nvars-1+1]; worky = new double[nclasses-1+1]; result = 0; for(i=0; i<=npoints-1; i++) { // // Process // for(i_=0; i_<=nvars-1;i_++) { workx[i_] = xy[i,i_]; } mnlprocess(lm, workx, ref worky, _params); // // Logit version of the answer // nmax = 0; for(j=0; j<=nclasses-1; j++) { if( (double)(worky[j])>(double)(worky[nmax]) ) { nmax = j; } } // // compare // if( nmax!=(int)Math.Round(xy[i,nvars]) ) { result = result+1; } } return result; } /************************************************************************* Internal subroutine. Places exponents of the anti-overflow shifted internal linear outputs into the service part of the W array. *************************************************************************/ private static void mnliexp(ref double[] w, double[] x, alglib.xparams _params) { int nvars = 0; int nclasses = 0; int offs = 0; int i = 0; int i1 = 0; double v = 0; double mx = 0; int i_ = 0; int i1_ = 0; alglib.ap.assert((double)(w[1])==(double)(logitvnum), "LOGIT: unexpected model version"); nvars = (int)Math.Round(w[2]); nclasses = (int)Math.Round(w[3]); offs = (int)Math.Round(w[4]); i1 = offs+(nvars+1)*(nclasses-1); for(i=0; i<=nclasses-2; i++) { i1_ = (0)-(offs+i*(nvars+1)); v = 0.0; for(i_=offs+i*(nvars+1); i_<=offs+i*(nvars+1)+nvars-1;i_++) { v += w[i_]*x[i_+i1_]; } w[i1+i] = v+w[offs+i*(nvars+1)+nvars]; } w[i1+nclasses-1] = 0; mx = 0; for(i=i1; i<=i1+nclasses-1; i++) { mx = Math.Max(mx, w[i]); } for(i=i1; i<=i1+nclasses-1; i++) { w[i] = Math.Exp(w[i]-mx); } } /************************************************************************* Calculation of all types of errors -- ALGLIB -- Copyright 30.08.2008 by Bochkanov Sergey *************************************************************************/ private static void mnlallerrors(logitmodel lm, double[,] xy, int npoints, ref double relcls, ref double avgce, ref double rms, ref double avg, ref double avgrel, alglib.xparams _params) { int nvars = 0; int nclasses = 0; int i = 0; double[] buf = new double[0]; double[] workx = new double[0]; double[] y = new double[0]; double[] dy = new double[0]; int i_ = 0; relcls = 0; avgce = 0; rms = 0; avg = 0; avgrel = 0; alglib.ap.assert((int)Math.Round(lm.w[1])==logitvnum, "MNL unit: Incorrect MNL version!"); nvars = (int)Math.Round(lm.w[2]); nclasses = (int)Math.Round(lm.w[3]); workx = new double[nvars-1+1]; y = new double[nclasses-1+1]; dy = new double[0+1]; bdss.dserrallocate(nclasses, ref buf, _params); for(i=0; i<=npoints-1; i++) { for(i_=0; i_<=nvars-1;i_++) { workx[i_] = xy[i,i_]; } mnlprocess(lm, workx, ref y, _params); dy[0] = xy[i,nvars]; bdss.dserraccumulate(ref buf, y, dy, _params); } bdss.dserrfinish(ref buf, _params); relcls = buf[0]; avgce = buf[1]; rms = buf[2]; avg = buf[3]; avgrel = buf[4]; } /************************************************************************* THE PURPOSE OF MCSRCH IS TO FIND A STEP WHICH SATISFIES A SUFFICIENT DECREASE CONDITION AND A CURVATURE CONDITION. AT EACH STAGE THE SUBROUTINE UPDATES AN INTERVAL OF UNCERTAINTY WITH ENDPOINTS STX AND STY. THE INTERVAL OF UNCERTAINTY IS INITIALLY CHOSEN SO THAT IT CONTAINS A MINIMIZER OF THE MODIFIED FUNCTION F(X+STP*S) - F(X) - FTOL*STP*(GRADF(X)'S). IF A STEP IS OBTAINED FOR WHICH THE MODIFIED FUNCTION HAS A NONPOSITIVE FUNCTION VALUE AND NONNEGATIVE DERIVATIVE, THEN THE INTERVAL OF UNCERTAINTY IS CHOSEN SO THAT IT CONTAINS A MINIMIZER OF F(X+STP*S). THE ALGORITHM IS DESIGNED TO FIND A STEP WHICH SATISFIES THE SUFFICIENT DECREASE CONDITION F(X+STP*S) .LE. F(X) + FTOL*STP*(GRADF(X)'S), AND THE CURVATURE CONDITION ABS(GRADF(X+STP*S)'S)) .LE. GTOL*ABS(GRADF(X)'S). IF FTOL IS LESS THAN GTOL AND IF, FOR EXAMPLE, THE FUNCTION IS BOUNDED BELOW, THEN THERE IS ALWAYS A STEP WHICH SATISFIES BOTH CONDITIONS. IF NO STEP CAN BE FOUND WHICH SATISFIES BOTH CONDITIONS, THEN THE ALGORITHM USUALLY STOPS WHEN ROUNDING ERRORS PREVENT FURTHER PROGRESS. IN THIS CASE STP ONLY SATISFIES THE SUFFICIENT DECREASE CONDITION. PARAMETERS DESCRIPRION N IS A POSITIVE INTEGER INPUT VARIABLE SET TO THE NUMBER OF VARIABLES. X IS AN ARRAY OF LENGTH N. ON INPUT IT MUST CONTAIN THE BASE POINT FOR THE LINE SEARCH. ON OUTPUT IT CONTAINS X+STP*S. F IS A VARIABLE. ON INPUT IT MUST CONTAIN THE VALUE OF F AT X. ON OUTPUT IT CONTAINS THE VALUE OF F AT X + STP*S. G IS AN ARRAY OF LENGTH N. ON INPUT IT MUST CONTAIN THE GRADIENT OF F AT X. ON OUTPUT IT CONTAINS THE GRADIENT OF F AT X + STP*S. S IS AN INPUT ARRAY OF LENGTH N WHICH SPECIFIES THE SEARCH DIRECTION. STP IS A NONNEGATIVE VARIABLE. ON INPUT STP CONTAINS AN INITIAL ESTIMATE OF A SATISFACTORY STEP. ON OUTPUT STP CONTAINS THE FINAL ESTIMATE. FTOL AND GTOL ARE NONNEGATIVE INPUT VARIABLES. TERMINATION OCCURS WHEN THE SUFFICIENT DECREASE CONDITION AND THE DIRECTIONAL DERIVATIVE CONDITION ARE SATISFIED. XTOL IS A NONNEGATIVE INPUT VARIABLE. TERMINATION OCCURS WHEN THE RELATIVE WIDTH OF THE INTERVAL OF UNCERTAINTY IS AT MOST XTOL. STPMIN AND STPMAX ARE NONNEGATIVE INPUT VARIABLES WHICH SPECIFY LOWER AND UPPER BOUNDS FOR THE STEP. MAXFEV IS A POSITIVE INTEGER INPUT VARIABLE. TERMINATION OCCURS WHEN THE NUMBER OF CALLS TO FCN IS AT LEAST MAXFEV BY THE END OF AN ITERATION. INFO IS AN INTEGER OUTPUT VARIABLE SET AS FOLLOWS: INFO = 0 IMPROPER INPUT PARAMETERS. INFO = 1 THE SUFFICIENT DECREASE CONDITION AND THE DIRECTIONAL DERIVATIVE CONDITION HOLD. INFO = 2 RELATIVE WIDTH OF THE INTERVAL OF UNCERTAINTY IS AT MOST XTOL. INFO = 3 NUMBER OF CALLS TO FCN HAS REACHED MAXFEV. INFO = 4 THE STEP IS AT THE LOWER BOUND STPMIN. INFO = 5 THE STEP IS AT THE UPPER BOUND STPMAX. INFO = 6 ROUNDING ERRORS PREVENT FURTHER PROGRESS. THERE MAY NOT BE A STEP WHICH SATISFIES THE SUFFICIENT DECREASE AND CURVATURE CONDITIONS. TOLERANCES MAY BE TOO SMALL. NFEV IS AN INTEGER OUTPUT VARIABLE SET TO THE NUMBER OF CALLS TO FCN. WA IS A WORK ARRAY OF LENGTH N. ARGONNE NATIONAL LABORATORY. MINPACK PROJECT. JUNE 1983 JORGE J. MORE', DAVID J. THUENTE *************************************************************************/ private static void mnlmcsrch(int n, ref double[] x, ref double f, ref double[] g, double[] s, ref double stp, ref int info, ref int nfev, ref double[] wa, logitmcstate state, ref int stage, alglib.xparams _params) { double v = 0; double p5 = 0; double p66 = 0; double zero = 0; int i_ = 0; // // init // p5 = 0.5; p66 = 0.66; state.xtrapf = 4.0; zero = 0; // // Main cycle // while( true ) { if( stage==0 ) { // // NEXT // stage = 2; continue; } if( stage==2 ) { state.infoc = 1; info = 0; // // CHECK THE INPUT PARAMETERS FOR ERRORS. // if( ((((((n<=0 || (double)(stp)<=(double)(0)) || (double)(ftol)<(double)(0)) || (double)(gtol)<(double)(zero)) || (double)(xtol)<(double)(zero)) || (double)(stpmin)<(double)(zero)) || (double)(stpmax)<(double)(stpmin)) || maxfev<=0 ) { stage = 0; return; } // // COMPUTE THE INITIAL GRADIENT IN THE SEARCH DIRECTION // AND CHECK THAT S IS A DESCENT DIRECTION. // v = 0.0; for(i_=0; i_<=n-1;i_++) { v += g[i_]*s[i_]; } state.dginit = v; if( (double)(state.dginit)>=(double)(0) ) { stage = 0; return; } // // INITIALIZE LOCAL VARIABLES. // state.brackt = false; state.stage1 = true; nfev = 0; state.finit = f; state.dgtest = ftol*state.dginit; state.width = stpmax-stpmin; state.width1 = state.width/p5; for(i_=0; i_<=n-1;i_++) { wa[i_] = x[i_]; } // // THE VARIABLES STX, FX, DGX CONTAIN THE VALUES OF THE STEP, // FUNCTION, AND DIRECTIONAL DERIVATIVE AT THE BEST STEP. // THE VARIABLES STY, FY, DGY CONTAIN THE VALUE OF THE STEP, // FUNCTION, AND DERIVATIVE AT THE OTHER ENDPOINT OF // THE INTERVAL OF UNCERTAINTY. // THE VARIABLES STP, F, DG CONTAIN THE VALUES OF THE STEP, // FUNCTION, AND DERIVATIVE AT THE CURRENT STEP. // state.stx = 0; state.fx = state.finit; state.dgx = state.dginit; state.sty = 0; state.fy = state.finit; state.dgy = state.dginit; // // NEXT // stage = 3; continue; } if( stage==3 ) { // // START OF ITERATION. // // SET THE MINIMUM AND MAXIMUM STEPS TO CORRESPOND // TO THE PRESENT INTERVAL OF UNCERTAINTY. // if( state.brackt ) { if( (double)(state.stx)<(double)(state.sty) ) { state.stmin = state.stx; state.stmax = state.sty; } else { state.stmin = state.sty; state.stmax = state.stx; } } else { state.stmin = state.stx; state.stmax = stp+state.xtrapf*(stp-state.stx); } // // FORCE THE STEP TO BE WITHIN THE BOUNDS STPMAX AND STPMIN. // if( (double)(stp)>(double)(stpmax) ) { stp = stpmax; } if( (double)(stp)<(double)(stpmin) ) { stp = stpmin; } // // IF AN UNUSUAL TERMINATION IS TO OCCUR THEN LET // STP BE THE LOWEST POINT OBTAINED SO FAR. // if( (((state.brackt && ((double)(stp)<=(double)(state.stmin) || (double)(stp)>=(double)(state.stmax))) || nfev>=maxfev-1) || state.infoc==0) || (state.brackt && (double)(state.stmax-state.stmin)<=(double)(xtol*state.stmax)) ) { stp = state.stx; } // // EVALUATE THE FUNCTION AND GRADIENT AT STP // AND COMPUTE THE DIRECTIONAL DERIVATIVE. // for(i_=0; i_<=n-1;i_++) { x[i_] = wa[i_]; } for(i_=0; i_<=n-1;i_++) { x[i_] = x[i_] + stp*s[i_]; } // // NEXT // stage = 4; return; } if( stage==4 ) { info = 0; nfev = nfev+1; v = 0.0; for(i_=0; i_<=n-1;i_++) { v += g[i_]*s[i_]; } state.dg = v; state.ftest1 = state.finit+stp*state.dgtest; // // TEST FOR CONVERGENCE. // if( (state.brackt && ((double)(stp)<=(double)(state.stmin) || (double)(stp)>=(double)(state.stmax))) || state.infoc==0 ) { info = 6; } if( ((double)(stp)==(double)(stpmax) && (double)(f)<=(double)(state.ftest1)) && (double)(state.dg)<=(double)(state.dgtest) ) { info = 5; } if( (double)(stp)==(double)(stpmin) && ((double)(f)>(double)(state.ftest1) || (double)(state.dg)>=(double)(state.dgtest)) ) { info = 4; } if( nfev>=maxfev ) { info = 3; } if( state.brackt && (double)(state.stmax-state.stmin)<=(double)(xtol*state.stmax) ) { info = 2; } if( (double)(f)<=(double)(state.ftest1) && (double)(Math.Abs(state.dg))<=(double)(-(gtol*state.dginit)) ) { info = 1; } // // CHECK FOR TERMINATION. // if( info!=0 ) { stage = 0; return; } // // IN THE FIRST STAGE WE SEEK A STEP FOR WHICH THE MODIFIED // FUNCTION HAS A NONPOSITIVE VALUE AND NONNEGATIVE DERIVATIVE. // if( (state.stage1 && (double)(f)<=(double)(state.ftest1)) && (double)(state.dg)>=(double)(Math.Min(ftol, gtol)*state.dginit) ) { state.stage1 = false; } // // A MODIFIED FUNCTION IS USED TO PREDICT THE STEP ONLY IF // WE HAVE NOT OBTAINED A STEP FOR WHICH THE MODIFIED // FUNCTION HAS A NONPOSITIVE FUNCTION VALUE AND NONNEGATIVE // DERIVATIVE, AND IF A LOWER FUNCTION VALUE HAS BEEN // OBTAINED BUT THE DECREASE IS NOT SUFFICIENT. // if( (state.stage1 && (double)(f)<=(double)(state.fx)) && (double)(f)>(double)(state.ftest1) ) { // // DEFINE THE MODIFIED FUNCTION AND DERIVATIVE VALUES. // state.fm = f-stp*state.dgtest; state.fxm = state.fx-state.stx*state.dgtest; state.fym = state.fy-state.sty*state.dgtest; state.dgm = state.dg-state.dgtest; state.dgxm = state.dgx-state.dgtest; state.dgym = state.dgy-state.dgtest; // // CALL CSTEP TO UPDATE THE INTERVAL OF UNCERTAINTY // AND TO COMPUTE THE NEW STEP. // mnlmcstep(ref state.stx, ref state.fxm, ref state.dgxm, ref state.sty, ref state.fym, ref state.dgym, ref stp, state.fm, state.dgm, ref state.brackt, state.stmin, state.stmax, ref state.infoc, _params); // // RESET THE FUNCTION AND GRADIENT VALUES FOR F. // state.fx = state.fxm+state.stx*state.dgtest; state.fy = state.fym+state.sty*state.dgtest; state.dgx = state.dgxm+state.dgtest; state.dgy = state.dgym+state.dgtest; } else { // // CALL MCSTEP TO UPDATE THE INTERVAL OF UNCERTAINTY // AND TO COMPUTE THE NEW STEP. // mnlmcstep(ref state.stx, ref state.fx, ref state.dgx, ref state.sty, ref state.fy, ref state.dgy, ref stp, f, state.dg, ref state.brackt, state.stmin, state.stmax, ref state.infoc, _params); } // // FORCE A SUFFICIENT DECREASE IN THE SIZE OF THE // INTERVAL OF UNCERTAINTY. // if( state.brackt ) { if( (double)(Math.Abs(state.sty-state.stx))>=(double)(p66*state.width1) ) { stp = state.stx+p5*(state.sty-state.stx); } state.width1 = state.width; state.width = Math.Abs(state.sty-state.stx); } // // NEXT. // stage = 3; continue; } } } private static void mnlmcstep(ref double stx, ref double fx, ref double dx, ref double sty, ref double fy, ref double dy, ref double stp, double fp, double dp, ref bool brackt, double stmin, double stmax, ref int info, alglib.xparams _params) { bool bound = new bool(); double gamma = 0; double p = 0; double q = 0; double r = 0; double s = 0; double sgnd = 0; double stpc = 0; double stpf = 0; double stpq = 0; double theta = 0; info = 0; // // CHECK THE INPUT PARAMETERS FOR ERRORS. // if( ((brackt && ((double)(stp)<=(double)(Math.Min(stx, sty)) || (double)(stp)>=(double)(Math.Max(stx, sty)))) || (double)(dx*(stp-stx))>=(double)(0)) || (double)(stmax)<(double)(stmin) ) { return; } // // DETERMINE IF THE DERIVATIVES HAVE OPPOSITE SIGN. // sgnd = dp*(dx/Math.Abs(dx)); // // FIRST CASE. A HIGHER FUNCTION VALUE. // THE MINIMUM IS BRACKETED. IF THE CUBIC STEP IS CLOSER // TO STX THAN THE QUADRATIC STEP, THE CUBIC STEP IS TAKEN, // ELSE THE AVERAGE OF THE CUBIC AND QUADRATIC STEPS IS TAKEN. // if( (double)(fp)>(double)(fx) ) { info = 1; bound = true; theta = 3*(fx-fp)/(stp-stx)+dx+dp; s = Math.Max(Math.Abs(theta), Math.Max(Math.Abs(dx), Math.Abs(dp))); gamma = s*Math.Sqrt(math.sqr(theta/s)-dx/s*(dp/s)); if( (double)(stp)<(double)(stx) ) { gamma = -gamma; } p = gamma-dx+theta; q = gamma-dx+gamma+dp; r = p/q; stpc = stx+r*(stp-stx); stpq = stx+dx/((fx-fp)/(stp-stx)+dx)/2*(stp-stx); if( (double)(Math.Abs(stpc-stx))<(double)(Math.Abs(stpq-stx)) ) { stpf = stpc; } else { stpf = stpc+(stpq-stpc)/2; } brackt = true; } else { if( (double)(sgnd)<(double)(0) ) { // // SECOND CASE. A LOWER FUNCTION VALUE AND DERIVATIVES OF // OPPOSITE SIGN. THE MINIMUM IS BRACKETED. IF THE CUBIC // STEP IS CLOSER TO STX THAN THE QUADRATIC (SECANT) STEP, // THE CUBIC STEP IS TAKEN, ELSE THE QUADRATIC STEP IS TAKEN. // info = 2; bound = false; theta = 3*(fx-fp)/(stp-stx)+dx+dp; s = Math.Max(Math.Abs(theta), Math.Max(Math.Abs(dx), Math.Abs(dp))); gamma = s*Math.Sqrt(math.sqr(theta/s)-dx/s*(dp/s)); if( (double)(stp)>(double)(stx) ) { gamma = -gamma; } p = gamma-dp+theta; q = gamma-dp+gamma+dx; r = p/q; stpc = stp+r*(stx-stp); stpq = stp+dp/(dp-dx)*(stx-stp); if( (double)(Math.Abs(stpc-stp))>(double)(Math.Abs(stpq-stp)) ) { stpf = stpc; } else { stpf = stpq; } brackt = true; } else { if( (double)(Math.Abs(dp))<(double)(Math.Abs(dx)) ) { // // THIRD CASE. A LOWER FUNCTION VALUE, DERIVATIVES OF THE // SAME SIGN, AND THE MAGNITUDE OF THE DERIVATIVE DECREASES. // THE CUBIC STEP IS ONLY USED IF THE CUBIC TENDS TO INFINITY // IN THE DIRECTION OF THE STEP OR IF THE MINIMUM OF THE CUBIC // IS BEYOND STP. OTHERWISE THE CUBIC STEP IS DEFINED TO BE // EITHER STPMIN OR STPMAX. THE QUADRATIC (SECANT) STEP IS ALSO // COMPUTED AND IF THE MINIMUM IS BRACKETED THEN THE THE STEP // CLOSEST TO STX IS TAKEN, ELSE THE STEP FARTHEST AWAY IS TAKEN. // info = 3; bound = true; theta = 3*(fx-fp)/(stp-stx)+dx+dp; s = Math.Max(Math.Abs(theta), Math.Max(Math.Abs(dx), Math.Abs(dp))); // // THE CASE GAMMA = 0 ONLY ARISES IF THE CUBIC DOES NOT TEND // TO INFINITY IN THE DIRECTION OF THE STEP. // gamma = s*Math.Sqrt(Math.Max(0, math.sqr(theta/s)-dx/s*(dp/s))); if( (double)(stp)>(double)(stx) ) { gamma = -gamma; } p = gamma-dp+theta; q = gamma+(dx-dp)+gamma; r = p/q; if( (double)(r)<(double)(0) && (double)(gamma)!=(double)(0) ) { stpc = stp+r*(stx-stp); } else { if( (double)(stp)>(double)(stx) ) { stpc = stmax; } else { stpc = stmin; } } stpq = stp+dp/(dp-dx)*(stx-stp); if( brackt ) { if( (double)(Math.Abs(stp-stpc))<(double)(Math.Abs(stp-stpq)) ) { stpf = stpc; } else { stpf = stpq; } } else { if( (double)(Math.Abs(stp-stpc))>(double)(Math.Abs(stp-stpq)) ) { stpf = stpc; } else { stpf = stpq; } } } else { // // FOURTH CASE. A LOWER FUNCTION VALUE, DERIVATIVES OF THE // SAME SIGN, AND THE MAGNITUDE OF THE DERIVATIVE DOES // NOT DECREASE. IF THE MINIMUM IS NOT BRACKETED, THE STEP // IS EITHER STPMIN OR STPMAX, ELSE THE CUBIC STEP IS TAKEN. // info = 4; bound = false; if( brackt ) { theta = 3*(fp-fy)/(sty-stp)+dy+dp; s = Math.Max(Math.Abs(theta), Math.Max(Math.Abs(dy), Math.Abs(dp))); gamma = s*Math.Sqrt(math.sqr(theta/s)-dy/s*(dp/s)); if( (double)(stp)>(double)(sty) ) { gamma = -gamma; } p = gamma-dp+theta; q = gamma-dp+gamma+dy; r = p/q; stpc = stp+r*(sty-stp); stpf = stpc; } else { if( (double)(stp)>(double)(stx) ) { stpf = stmax; } else { stpf = stmin; } } } } } // // UPDATE THE INTERVAL OF UNCERTAINTY. THIS UPDATE DOES NOT // DEPEND ON THE NEW STEP OR THE CASE ANALYSIS ABOVE. // if( (double)(fp)>(double)(fx) ) { sty = stp; fy = fp; dy = dp; } else { if( (double)(sgnd)<(double)(0.0) ) { sty = stx; fy = fx; dy = dx; } stx = stp; fx = fp; dx = dp; } // // COMPUTE THE NEW STEP AND SAFEGUARD IT. // stpf = Math.Min(stmax, stpf); stpf = Math.Max(stmin, stpf); stp = stpf; if( brackt && bound ) { if( (double)(sty)>(double)(stx) ) { stp = Math.Min(stx+0.66*(sty-stx), stp); } else { stp = Math.Max(stx+0.66*(sty-stx), stp); } } } } public class mcpd { /************************************************************************* This structure is a MCPD (Markov Chains for Population Data) solver. You should use ALGLIB functions in order to work with this object. -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public class mcpdstate : apobject { public int n; public int[] states; public int npairs; public double[,] data; public double[,] ec; public double[,] bndl; public double[,] bndu; public double[,] c; public int[] ct; public int ccnt; public double[] pw; public double[,] priorp; public double regterm; public minbleic.minbleicstate bs; public int repinneriterationscount; public int repouteriterationscount; public int repnfev; public int repterminationtype; public minbleic.minbleicreport br; public double[] tmpp; public double[] effectivew; public double[] effectivebndl; public double[] effectivebndu; public double[,] effectivec; public int[] effectivect; public double[] h; public double[,] p; public mcpdstate() { init(); } public override void init() { states = new int[0]; data = new double[0,0]; ec = new double[0,0]; bndl = new double[0,0]; bndu = new double[0,0]; c = new double[0,0]; ct = new int[0]; pw = new double[0]; priorp = new double[0,0]; bs = new minbleic.minbleicstate(); br = new minbleic.minbleicreport(); tmpp = new double[0]; effectivew = new double[0]; effectivebndl = new double[0]; effectivebndu = new double[0]; effectivec = new double[0,0]; effectivect = new int[0]; h = new double[0]; p = new double[0,0]; } public override alglib.apobject make_copy() { mcpdstate _result = new mcpdstate(); _result.n = n; _result.states = (int[])states.Clone(); _result.npairs = npairs; _result.data = (double[,])data.Clone(); _result.ec = (double[,])ec.Clone(); _result.bndl = (double[,])bndl.Clone(); _result.bndu = (double[,])bndu.Clone(); _result.c = (double[,])c.Clone(); _result.ct = (int[])ct.Clone(); _result.ccnt = ccnt; _result.pw = (double[])pw.Clone(); _result.priorp = (double[,])priorp.Clone(); _result.regterm = regterm; _result.bs = (minbleic.minbleicstate)bs.make_copy(); _result.repinneriterationscount = repinneriterationscount; _result.repouteriterationscount = repouteriterationscount; _result.repnfev = repnfev; _result.repterminationtype = repterminationtype; _result.br = (minbleic.minbleicreport)br.make_copy(); _result.tmpp = (double[])tmpp.Clone(); _result.effectivew = (double[])effectivew.Clone(); _result.effectivebndl = (double[])effectivebndl.Clone(); _result.effectivebndu = (double[])effectivebndu.Clone(); _result.effectivec = (double[,])effectivec.Clone(); _result.effectivect = (int[])effectivect.Clone(); _result.h = (double[])h.Clone(); _result.p = (double[,])p.Clone(); return _result; } }; /************************************************************************* This structure is a MCPD training report: InnerIterationsCount - number of inner iterations of the underlying optimization algorithm OuterIterationsCount - number of outer iterations of the underlying optimization algorithm NFEV - number of merit function evaluations TerminationType - termination type (same as for MinBLEIC optimizer, positive values denote success, negative ones - failure) -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public class mcpdreport : apobject { public int inneriterationscount; public int outeriterationscount; public int nfev; public int terminationtype; public mcpdreport() { init(); } public override void init() { } public override alglib.apobject make_copy() { mcpdreport _result = new mcpdreport(); _result.inneriterationscount = inneriterationscount; _result.outeriterationscount = outeriterationscount; _result.nfev = nfev; _result.terminationtype = terminationtype; return _result; } }; public const double xtol = 1.0E-8; /************************************************************************* DESCRIPTION: This function creates MCPD (Markov Chains for Population Data) solver. This solver can be used to find transition matrix P for N-dimensional prediction problem where transition from X[i] to X[i+1] is modelled as X[i+1] = P*X[i] where X[i] and X[i+1] are N-dimensional population vectors (components of each X are non-negative), and P is a N*N transition matrix (elements of P are non-negative, each column sums to 1.0). Such models arise when when: * there is some population of individuals * individuals can have different states * individuals can transit from one state to another * population size is constant, i.e. there is no new individuals and no one leaves population * you want to model transitions of individuals from one state into another USAGE: Here we give very brief outline of the MCPD. We strongly recommend you to read examples in the ALGLIB Reference Manual and to read ALGLIB User Guide on data analysis which is available at http://www.alglib.net/dataanalysis/ 1. User initializes algorithm state with MCPDCreate() call 2. User adds one or more tracks - sequences of states which describe evolution of a system being modelled from different starting conditions 3. User may add optional boundary, equality and/or linear constraints on the coefficients of P by calling one of the following functions: * MCPDSetEC() to set equality constraints * MCPDSetBC() to set bound constraints * MCPDSetLC() to set linear constraints 4. Optionally, user may set custom weights for prediction errors (by default, algorithm assigns non-equal, automatically chosen weights for errors in the prediction of different components of X). It can be done with a call of MCPDSetPredictionWeights() function. 5. User calls MCPDSolve() function which takes algorithm state and pointer (delegate, etc.) to callback function which calculates F/G. 6. User calls MCPDResults() to get solution INPUT PARAMETERS: N - problem dimension, N>=1 OUTPUT PARAMETERS: State - structure stores algorithm state -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdcreate(int n, mcpdstate s, alglib.xparams _params) { alglib.ap.assert(n>=1, "MCPDCreate: N<1"); mcpdinit(n, -1, -1, s, _params); } /************************************************************************* DESCRIPTION: This function is a specialized version of MCPDCreate() function, and we recommend you to read comments for this function for general information about MCPD solver. This function creates MCPD (Markov Chains for Population Data) solver for "Entry-state" model, i.e. model where transition from X[i] to X[i+1] is modelled as X[i+1] = P*X[i] where X[i] and X[i+1] are N-dimensional state vectors P is a N*N transition matrix and one selected component of X[] is called "entry" state and is treated in a special way: system state always transits from "entry" state to some another state system state can not transit from any state into "entry" state Such conditions basically mean that row of P which corresponds to "entry" state is zero. Such models arise when: * there is some population of individuals * individuals can have different states * individuals can transit from one state to another * population size is NOT constant - at every moment of time there is some (unpredictable) amount of "new" individuals, which can transit into one of the states at the next turn, but still no one leaves population * you want to model transitions of individuals from one state into another * but you do NOT want to predict amount of "new" individuals because it does not depends on individuals already present (hence system can not transit INTO entry state - it can only transit FROM it). This model is discussed in more details in the ALGLIB User Guide (see http://www.alglib.net/dataanalysis/ for more data). INPUT PARAMETERS: N - problem dimension, N>=2 EntryState- index of entry state, in 0..N-1 OUTPUT PARAMETERS: State - structure stores algorithm state -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdcreateentry(int n, int entrystate, mcpdstate s, alglib.xparams _params) { alglib.ap.assert(n>=2, "MCPDCreateEntry: N<2"); alglib.ap.assert(entrystate>=0, "MCPDCreateEntry: EntryState<0"); alglib.ap.assert(entrystate=N"); mcpdinit(n, entrystate, -1, s, _params); } /************************************************************************* DESCRIPTION: This function is a specialized version of MCPDCreate() function, and we recommend you to read comments for this function for general information about MCPD solver. This function creates MCPD (Markov Chains for Population Data) solver for "Exit-state" model, i.e. model where transition from X[i] to X[i+1] is modelled as X[i+1] = P*X[i] where X[i] and X[i+1] are N-dimensional state vectors P is a N*N transition matrix and one selected component of X[] is called "exit" state and is treated in a special way: system state can transit from any state into "exit" state system state can not transit from "exit" state into any other state transition operator discards "exit" state (makes it zero at each turn) Such conditions basically mean that column of P which corresponds to "exit" state is zero. Multiplication by such P may decrease sum of vector components. Such models arise when: * there is some population of individuals * individuals can have different states * individuals can transit from one state to another * population size is NOT constant - individuals can move into "exit" state and leave population at the next turn, but there are no new individuals * amount of individuals which leave population can be predicted * you want to model transitions of individuals from one state into another (including transitions into the "exit" state) This model is discussed in more details in the ALGLIB User Guide (see http://www.alglib.net/dataanalysis/ for more data). INPUT PARAMETERS: N - problem dimension, N>=2 ExitState- index of exit state, in 0..N-1 OUTPUT PARAMETERS: State - structure stores algorithm state -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdcreateexit(int n, int exitstate, mcpdstate s, alglib.xparams _params) { alglib.ap.assert(n>=2, "MCPDCreateExit: N<2"); alglib.ap.assert(exitstate>=0, "MCPDCreateExit: ExitState<0"); alglib.ap.assert(exitstate=N"); mcpdinit(n, -1, exitstate, s, _params); } /************************************************************************* DESCRIPTION: This function is a specialized version of MCPDCreate() function, and we recommend you to read comments for this function for general information about MCPD solver. This function creates MCPD (Markov Chains for Population Data) solver for "Entry-Exit-states" model, i.e. model where transition from X[i] to X[i+1] is modelled as X[i+1] = P*X[i] where X[i] and X[i+1] are N-dimensional state vectors P is a N*N transition matrix one selected component of X[] is called "entry" state and is treated in a special way: system state always transits from "entry" state to some another state system state can not transit from any state into "entry" state and another one component of X[] is called "exit" state and is treated in a special way too: system state can transit from any state into "exit" state system state can not transit from "exit" state into any other state transition operator discards "exit" state (makes it zero at each turn) Such conditions basically mean that: row of P which corresponds to "entry" state is zero column of P which corresponds to "exit" state is zero Multiplication by such P may decrease sum of vector components. Such models arise when: * there is some population of individuals * individuals can have different states * individuals can transit from one state to another * population size is NOT constant * at every moment of time there is some (unpredictable) amount of "new" individuals, which can transit into one of the states at the next turn * some individuals can move (predictably) into "exit" state and leave population at the next turn * you want to model transitions of individuals from one state into another, including transitions from the "entry" state and into the "exit" state. * but you do NOT want to predict amount of "new" individuals because it does not depends on individuals already present (hence system can not transit INTO entry state - it can only transit FROM it). This model is discussed in more details in the ALGLIB User Guide (see http://www.alglib.net/dataanalysis/ for more data). INPUT PARAMETERS: N - problem dimension, N>=2 EntryState- index of entry state, in 0..N-1 ExitState- index of exit state, in 0..N-1 OUTPUT PARAMETERS: State - structure stores algorithm state -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdcreateentryexit(int n, int entrystate, int exitstate, mcpdstate s, alglib.xparams _params) { alglib.ap.assert(n>=2, "MCPDCreateEntryExit: N<2"); alglib.ap.assert(entrystate>=0, "MCPDCreateEntryExit: EntryState<0"); alglib.ap.assert(entrystate=N"); alglib.ap.assert(exitstate>=0, "MCPDCreateEntryExit: ExitState<0"); alglib.ap.assert(exitstate=N"); alglib.ap.assert(entrystate!=exitstate, "MCPDCreateEntryExit: EntryState=ExitState"); mcpdinit(n, entrystate, exitstate, s, _params); } /************************************************************************* This function is used to add a track - sequence of system states at the different moments of its evolution. You may add one or several tracks to the MCPD solver. In case you have several tracks, they won't overwrite each other. For example, if you pass two tracks, A1-A2-A3 (system at t=A+1, t=A+2 and t=A+3) and B1-B2-B3, then solver will try to model transitions from t=A+1 to t=A+2, t=A+2 to t=A+3, t=B+1 to t=B+2, t=B+2 to t=B+3. But it WONT mix these two tracks - i.e. it wont try to model transition from t=A+3 to t=B+1. INPUT PARAMETERS: S - solver XY - track, array[K,N]: * I-th row is a state at t=I * elements of XY must be non-negative (exception will be thrown on negative elements) K - number of points in a track * if given, only leading K rows of XY are used * if not given, automatically determined from size of XY NOTES: 1. Track may contain either proportional or population data: * with proportional data all rows of XY must sum to 1.0, i.e. we have proportions instead of absolute population values * with population data rows of XY contain population counts and generally do not sum to 1.0 (although they still must be non-negative) -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdaddtrack(mcpdstate s, double[,] xy, int k, alglib.xparams _params) { int i = 0; int j = 0; int n = 0; double s0 = 0; double s1 = 0; n = s.n; alglib.ap.assert(k>=0, "MCPDAddTrack: K<0"); alglib.ap.assert(alglib.ap.cols(xy)>=n, "MCPDAddTrack: Cols(XY)=k, "MCPDAddTrack: Rows(XY)=(double)(0), "MCPDAddTrack: XY contains negative elements"); } } if( k<2 ) { return; } if( alglib.ap.rows(s.data)=0 ) { s0 = s0+xy[i,j]; } if( s.states[j]<=0 ) { s1 = s1+xy[i+1,j]; } } if( (double)(s0)>(double)(0) && (double)(s1)>(double)(0) ) { for(j=0; j<=n-1; j++) { if( s.states[j]>=0 ) { s.data[s.npairs,j] = xy[i,j]/s0; } else { s.data[s.npairs,j] = 0.0; } if( s.states[j]<=0 ) { s.data[s.npairs,n+j] = xy[i+1,j]/s1; } else { s.data[s.npairs,n+j] = 0.0; } } s.npairs = s.npairs+1; } } } /************************************************************************* This function is used to add equality constraints on the elements of the transition matrix P. MCPD solver has four types of constraints which can be placed on P: * user-specified equality constraints (optional) * user-specified bound constraints (optional) * user-specified general linear constraints (optional) * basic constraints (always present): * non-negativity: P[i,j]>=0 * consistency: every column of P sums to 1.0 Final constraints which are passed to the underlying optimizer are calculated as intersection of all present constraints. For example, you may specify boundary constraint on P[0,0] and equality one: 0.1<=P[0,0]<=0.9 P[0,0]=0.5 Such combination of constraints will be silently reduced to their intersection, which is P[0,0]=0.5. This function can be used to place equality constraints on arbitrary subset of elements of P. Set of constraints is specified by EC, which may contain either NAN's or finite numbers from [0,1]. NAN denotes absence of constraint, finite number denotes equality constraint on specific element of P. You can also use MCPDAddEC() function which allows to ADD equality constraint for one element of P without changing constraints for other elements. These functions (MCPDSetEC and MCPDAddEC) interact as follows: * there is internal matrix of equality constraints which is stored in the MCPD solver * MCPDSetEC() replaces this matrix by another one (SET) * MCPDAddEC() modifies one element of this matrix and leaves other ones unchanged (ADD) * thus MCPDAddEC() call preserves all modifications done by previous calls, while MCPDSetEC() completely discards all changes done to the equality constraints. INPUT PARAMETERS: S - solver EC - equality constraints, array[N,N]. Elements of EC can be either NAN's or finite numbers from [0,1]. NAN denotes absence of constraints, while finite value denotes equality constraint on the corresponding element of P. NOTES: 1. infinite values of EC will lead to exception being thrown. Values less than 0.0 or greater than 1.0 will lead to error code being returned after call to MCPDSolve(). -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsetec(mcpdstate s, double[,] ec, alglib.xparams _params) { int i = 0; int j = 0; int n = 0; n = s.n; alglib.ap.assert(alglib.ap.cols(ec)>=n, "MCPDSetEC: Cols(EC)=n, "MCPDSetEC: Rows(EC)=0 * consistency: every column of P sums to 1.0 Final constraints which are passed to the underlying optimizer are calculated as intersection of all present constraints. For example, you may specify boundary constraint on P[0,0] and equality one: 0.1<=P[0,0]<=0.9 P[0,0]=0.5 Such combination of constraints will be silently reduced to their intersection, which is P[0,0]=0.5. This function can be used to ADD equality constraint for one element of P without changing constraints for other elements. You can also use MCPDSetEC() function which allows you to specify arbitrary set of equality constraints in one call. These functions (MCPDSetEC and MCPDAddEC) interact as follows: * there is internal matrix of equality constraints which is stored in the MCPD solver * MCPDSetEC() replaces this matrix by another one (SET) * MCPDAddEC() modifies one element of this matrix and leaves other ones unchanged (ADD) * thus MCPDAddEC() call preserves all modifications done by previous calls, while MCPDSetEC() completely discards all changes done to the equality constraints. INPUT PARAMETERS: S - solver I - row index of element being constrained J - column index of element being constrained C - value (constraint for P[I,J]). Can be either NAN (no constraint) or finite value from [0,1]. NOTES: 1. infinite values of C will lead to exception being thrown. Values less than 0.0 or greater than 1.0 will lead to error code being returned after call to MCPDSolve(). -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdaddec(mcpdstate s, int i, int j, double c, alglib.xparams _params) { alglib.ap.assert(i>=0, "MCPDAddEC: I<0"); alglib.ap.assert(i=N"); alglib.ap.assert(j>=0, "MCPDAddEC: J<0"); alglib.ap.assert(j=N"); alglib.ap.assert(Double.IsNaN(c) || math.isfinite(c), "MCPDAddEC: C is not finite number or NAN"); s.ec[i,j] = c; } /************************************************************************* This function is used to add bound constraints on the elements of the transition matrix P. MCPD solver has four types of constraints which can be placed on P: * user-specified equality constraints (optional) * user-specified bound constraints (optional) * user-specified general linear constraints (optional) * basic constraints (always present): * non-negativity: P[i,j]>=0 * consistency: every column of P sums to 1.0 Final constraints which are passed to the underlying optimizer are calculated as intersection of all present constraints. For example, you may specify boundary constraint on P[0,0] and equality one: 0.1<=P[0,0]<=0.9 P[0,0]=0.5 Such combination of constraints will be silently reduced to their intersection, which is P[0,0]=0.5. This function can be used to place bound constraints on arbitrary subset of elements of P. Set of constraints is specified by BndL/BndU matrices, which may contain arbitrary combination of finite numbers or infinities (like -INF=n, "MCPDSetBC: Cols(BndL)=n, "MCPDSetBC: Rows(BndL)=n, "MCPDSetBC: Cols(BndU)=n, "MCPDSetBC: Rows(BndU)=0 * consistency: every column of P sums to 1.0 Final constraints which are passed to the underlying optimizer are calculated as intersection of all present constraints. For example, you may specify boundary constraint on P[0,0] and equality one: 0.1<=P[0,0]<=0.9 P[0,0]=0.5 Such combination of constraints will be silently reduced to their intersection, which is P[0,0]=0.5. This function can be used to ADD bound constraint for one element of P without changing constraints for other elements. You can also use MCPDSetBC() function which allows to place bound constraints on arbitrary subset of elements of P. Set of constraints is specified by BndL/BndU matrices, which may contain arbitrary combination of finite numbers or infinities (like -INF=0, "MCPDAddBC: I<0"); alglib.ap.assert(i=N"); alglib.ap.assert(j>=0, "MCPDAddBC: J<0"); alglib.ap.assert(j=N"); alglib.ap.assert(math.isfinite(bndl) || Double.IsNegativeInfinity(bndl), "MCPDAddBC: BndL is NAN or +INF"); alglib.ap.assert(math.isfinite(bndu) || Double.IsPositiveInfinity(bndu), "MCPDAddBC: BndU is NAN or -INF"); s.bndl[i,j] = bndl; s.bndu[i,j] = bndu; } /************************************************************************* This function is used to set linear equality/inequality constraints on the elements of the transition matrix P. This function can be used to set one or several general linear constraints on the elements of P. Two types of constraints are supported: * equality constraints * inequality constraints (both less-or-equal and greater-or-equal) Coefficients of constraints are specified by matrix C (one of the parameters). One row of C corresponds to one constraint. Because transition matrix P has N*N elements, we need N*N columns to store all coefficients (they are stored row by row), and one more column to store right part - hence C has N*N+1 columns. Constraint kind is stored in the CT array. Thus, I-th linear constraint is P[0,0]*C[I,0] + P[0,1]*C[I,1] + .. + P[0,N-1]*C[I,N-1] + + P[1,0]*C[I,N] + P[1,1]*C[I,N+1] + ... + + P[N-1,N-1]*C[I,N*N-1] ?=? C[I,N*N] where ?=? can be either "=" (CT[i]=0), "<=" (CT[i]<0) or ">=" (CT[i]>0). Your constraint may involve only some subset of P (less than N*N elements). For example it can be something like P[0,0] + P[0,1] = 0.5 In this case you still should pass matrix with N*N+1 columns, but all its elements (except for C[0,0], C[0,1] and C[0,N*N-1]) will be zero. INPUT PARAMETERS: S - solver C - array[K,N*N+1] - coefficients of constraints (see above for complete description) CT - array[K] - constraint types (see above for complete description) K - number of equality/inequality constraints, K>=0: * if given, only leading K elements of C/CT are used * if not given, automatically determined from sizes of C/CT -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsetlc(mcpdstate s, double[,] c, int[] ct, int k, alglib.xparams _params) { int i = 0; int j = 0; int n = 0; n = s.n; alglib.ap.assert(alglib.ap.cols(c)>=n*n+1, "MCPDSetLC: Cols(C)=k, "MCPDSetLC: Rows(C)=k, "MCPDSetLC: Len(CT)=(double)(0.0), "MCPDSetTikhonovRegularizer: V is less than zero"); s.regterm = v; } /************************************************************************* This function allows to set prior values used for regularization of your problem. By default, regularizing term is equal to r*||P-prior_P||^2, where r is a small non-zero value, P is transition matrix, prior_P is identity matrix, ||X||^2 is a sum of squared elements of X. This function allows you to change prior values prior_P. You can also change r with MCPDSetTikhonovRegularizer() function. INPUT PARAMETERS: S - solver PP - array[N,N], matrix of prior values: 1. elements must be real numbers from [0,1] 2. columns must sum to 1.0. First property is checked (exception is thrown otherwise), while second one is not checked/enforced. -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsetprior(mcpdstate s, double[,] pp, alglib.xparams _params) { int i = 0; int j = 0; int n = 0; pp = (double[,])pp.Clone(); n = s.n; alglib.ap.assert(alglib.ap.cols(pp)>=n, "MCPDSetPrior: Cols(PP)=n, "MCPDSetPrior: Rows(PP)=(double)(0.0) && (double)(pp[i,j])<=(double)(1.0), "MCPDSetPrior: PP[i,j] is less than 0.0 or greater than 1.0"); s.priorp[i,j] = pp[i,j]; } } } /************************************************************************* This function is used to change prediction weights MCPD solver scales prediction errors as follows Error(P) = ||W*(y-P*x)||^2 where x is a system state at time t y is a system state at time t+1 P is a transition matrix W is a diagonal scaling matrix By default, weights are chosen in order to minimize relative prediction error instead of absolute one. For example, if one component of state is about 0.5 in magnitude and another one is about 0.05, then algorithm will make corresponding weights equal to 2.0 and 20.0. INPUT PARAMETERS: S - solver PW - array[N], weights: * must be non-negative values (exception will be thrown otherwise) * zero values will be replaced by automatically chosen values -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsetpredictionweights(mcpdstate s, double[] pw, alglib.xparams _params) { int i = 0; int n = 0; n = s.n; alglib.ap.assert(alglib.ap.len(pw)>=n, "MCPDSetPredictionWeights: Length(PW)=(double)(0), "MCPDSetPredictionWeights: PW containts negative elements"); s.pw[i] = pw[i]; } } /************************************************************************* This function is used to start solution of the MCPD problem. After return from this function, you can use MCPDResults() to get solution and completion code. -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdsolve(mcpdstate s, alglib.xparams _params) { int n = 0; int npairs = 0; int ccnt = 0; int i = 0; int j = 0; int k = 0; int k2 = 0; double v = 0; double vv = 0; int i_ = 0; int i1_ = 0; n = s.n; npairs = s.npairs; // // init fields of S // s.repterminationtype = 0; s.repinneriterationscount = 0; s.repouteriterationscount = 0; s.repnfev = 0; for(k=0; k<=n-1; k++) { for(k2=0; k2<=n-1; k2++) { s.p[k,k2] = Double.NaN; } } // // Generate "effective" weights for prediction and calculate preconditioner // for(i=0; i<=n-1; i++) { if( (double)(s.pw[i])==(double)(0) ) { v = 0; k = 0; for(j=0; j<=npairs-1; j++) { if( (double)(s.data[j,n+i])!=(double)(0) ) { v = v+s.data[j,n+i]; k = k+1; } } if( k!=0 ) { s.effectivew[i] = k/v; } else { s.effectivew[i] = 1.0; } } else { s.effectivew[i] = s.pw[i]; } } for(i=0; i<=n-1; i++) { for(j=0; j<=n-1; j++) { s.h[i*n+j] = 2*s.regterm; } } for(k=0; k<=npairs-1; k++) { for(i=0; i<=n-1; i++) { for(j=0; j<=n-1; j++) { s.h[i*n+j] = s.h[i*n+j]+2*math.sqr(s.effectivew[i])*math.sqr(s.data[k,j]); } } } for(i=0; i<=n-1; i++) { for(j=0; j<=n-1; j++) { if( (double)(s.h[i*n+j])==(double)(0) ) { s.h[i*n+j] = 1; } } } // // Generate "effective" BndL/BndU // for(i=0; i<=n-1; i++) { for(j=0; j<=n-1; j++) { // // Set default boundary constraints. // Lower bound is always zero, upper bound is calculated // with respect to entry/exit states. // s.effectivebndl[i*n+j] = 0.0; if( s.states[i]>0 || s.states[j]<0 ) { s.effectivebndu[i*n+j] = 0.0; } else { s.effectivebndu[i*n+j] = 1.0; } // // Calculate intersection of the default and user-specified bound constraints. // This code checks consistency of such combination. // if( math.isfinite(s.bndl[i,j]) && (double)(s.bndl[i,j])>(double)(s.effectivebndl[i*n+j]) ) { s.effectivebndl[i*n+j] = s.bndl[i,j]; } if( math.isfinite(s.bndu[i,j]) && (double)(s.bndu[i,j])<(double)(s.effectivebndu[i*n+j]) ) { s.effectivebndu[i*n+j] = s.bndu[i,j]; } if( (double)(s.effectivebndl[i*n+j])>(double)(s.effectivebndu[i*n+j]) ) { s.repterminationtype = -3; return; } // // Calculate intersection of the effective bound constraints // and user-specified equality constraints. // This code checks consistency of such combination. // if( math.isfinite(s.ec[i,j]) ) { if( (double)(s.ec[i,j])<(double)(s.effectivebndl[i*n+j]) || (double)(s.ec[i,j])>(double)(s.effectivebndu[i*n+j]) ) { s.repterminationtype = -3; return; } s.effectivebndl[i*n+j] = s.ec[i,j]; s.effectivebndu[i*n+j] = s.ec[i,j]; } } } // // Generate linear constraints: // * "default" sums-to-one constraints (not generated for "exit" states) // apserv.rmatrixsetlengthatleast(ref s.effectivec, s.ccnt+n, n*n+1, _params); apserv.ivectorsetlengthatleast(ref s.effectivect, s.ccnt+n, _params); ccnt = s.ccnt; for(i=0; i<=s.ccnt-1; i++) { for(j=0; j<=n*n; j++) { s.effectivec[i,j] = s.c[i,j]; } s.effectivect[i] = s.ct[i]; } for(i=0; i<=n-1; i++) { if( s.states[i]>=0 ) { for(k=0; k<=n*n-1; k++) { s.effectivec[ccnt,k] = 0; } for(k=0; k<=n-1; k++) { s.effectivec[ccnt,k*n+i] = 1; } s.effectivec[ccnt,n*n] = 1.0; s.effectivect[ccnt] = 0; ccnt = ccnt+1; } } // // create optimizer // for(i=0; i<=n-1; i++) { for(j=0; j<=n-1; j++) { s.tmpp[i*n+j] = (double)1/(double)n; } } minbleic.minbleicrestartfrom(s.bs, s.tmpp, _params); minbleic.minbleicsetbc(s.bs, s.effectivebndl, s.effectivebndu, _params); minbleic.minbleicsetlc(s.bs, s.effectivec, s.effectivect, ccnt, _params); minbleic.minbleicsetcond(s.bs, 0.0, 0.0, xtol, 0, _params); minbleic.minbleicsetprecdiag(s.bs, s.h, _params); // // solve problem // while( minbleic.minbleiciteration(s.bs, _params) ) { alglib.ap.assert(s.bs.needfg, "MCPDSolve: internal error"); if( s.bs.needfg ) { // // Calculate regularization term // s.bs.f = 0.0; vv = s.regterm; for(i=0; i<=n-1; i++) { for(j=0; j<=n-1; j++) { s.bs.f = s.bs.f+vv*math.sqr(s.bs.x[i*n+j]-s.priorp[i,j]); s.bs.g[i*n+j] = 2*vv*(s.bs.x[i*n+j]-s.priorp[i,j]); } } // // calculate prediction error/gradient for K-th pair // for(k=0; k<=npairs-1; k++) { for(i=0; i<=n-1; i++) { i1_ = (0)-(i*n); v = 0.0; for(i_=i*n; i_<=i*n+n-1;i_++) { v += s.bs.x[i_]*s.data[k,i_+i1_]; } vv = s.effectivew[i]; s.bs.f = s.bs.f+math.sqr(vv*(v-s.data[k,n+i])); for(j=0; j<=n-1; j++) { s.bs.g[i*n+j] = s.bs.g[i*n+j]+2*vv*vv*(v-s.data[k,n+i])*s.data[k,j]; } } } // // continue // continue; } } minbleic.minbleicresultsbuf(s.bs, ref s.tmpp, s.br, _params); for(i=0; i<=n-1; i++) { for(j=0; j<=n-1; j++) { s.p[i,j] = s.tmpp[i*n+j]; } } s.repterminationtype = s.br.terminationtype; s.repinneriterationscount = s.br.inneriterationscount; s.repouteriterationscount = s.br.outeriterationscount; s.repnfev = s.br.nfev; } /************************************************************************* MCPD results INPUT PARAMETERS: State - algorithm state OUTPUT PARAMETERS: P - array[N,N], transition matrix Rep - optimization report. You should check Rep.TerminationType in order to distinguish successful termination from unsuccessful one. Speaking short, positive values denote success, negative ones are failures. More information about fields of this structure can be found in the comments on MCPDReport datatype. -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ public static void mcpdresults(mcpdstate s, ref double[,] p, mcpdreport rep, alglib.xparams _params) { int i = 0; int j = 0; p = new double[0,0]; p = new double[s.n, s.n]; for(i=0; i<=s.n-1; i++) { for(j=0; j<=s.n-1; j++) { p[i,j] = s.p[i,j]; } } rep.terminationtype = s.repterminationtype; rep.inneriterationscount = s.repinneriterationscount; rep.outeriterationscount = s.repouteriterationscount; rep.nfev = s.repnfev; } /************************************************************************* Internal initialization function -- ALGLIB -- Copyright 23.05.2010 by Bochkanov Sergey *************************************************************************/ private static void mcpdinit(int n, int entrystate, int exitstate, mcpdstate s, alglib.xparams _params) { int i = 0; int j = 0; alglib.ap.assert(n>=1, "MCPDCreate: N<1"); s.n = n; s.states = new int[n]; for(i=0; i<=n-1; i++) { s.states[i] = 0; } if( entrystate>=0 ) { s.states[entrystate] = 1; } if( exitstate>=0 ) { s.states[exitstate] = -1; } s.npairs = 0; s.regterm = 1.0E-8; s.ccnt = 0; s.p = new double[n, n]; s.ec = new double[n, n]; s.bndl = new double[n, n]; s.bndu = new double[n, n]; s.pw = new double[n]; s.priorp = new double[n, n]; s.tmpp = new double[n*n]; s.effectivew = new double[n]; s.effectivebndl = new double[n*n]; s.effectivebndu = new double[n*n]; s.h = new double[n*n]; for(i=0; i<=n-1; i++) { for(j=0; j<=n-1; j++) { s.p[i,j] = 0.0; s.priorp[i,j] = 0.0; s.bndl[i,j] = Double.NegativeInfinity; s.bndu[i,j] = Double.PositiveInfinity; s.ec[i,j] = Double.NaN; } s.pw[i] = 0.0; s.priorp[i,i] = 1.0; } s.data = new double[1, 2*n]; for(i=0; i<=2*n-1; i++) { s.data[0,i] = 0.0; } for(i=0; i<=n*n-1; i++) { s.tmpp[i] = 0.0; } minbleic.minbleiccreate(n*n, s.tmpp, s.bs, _params); } } public class mlpe { /************************************************************************* Neural networks ensemble *************************************************************************/ public class mlpensemble : apobject { public int ensemblesize; public double[] weights; public double[] columnmeans; public double[] columnsigmas; public mlpbase.multilayerperceptron network; public double[] y; public mlpensemble() { init(); } public override void init() { weights = new double[0]; columnmeans = new double[0]; columnsigmas = new double[0]; network = new mlpbase.multilayerperceptron(); y = new double[0]; } public override alglib.apobject make_copy() { mlpensemble _result = new mlpensemble(); _result.ensemblesize = ensemblesize; _result.weights = (double[])weights.Clone(); _result.columnmeans = (double[])columnmeans.Clone(); _result.columnsigmas = (double[])columnsigmas.Clone(); _result.network = (mlpbase.multilayerperceptron)network.make_copy(); _result.y = (double[])y.Clone(); return _result; } }; public const int mlpefirstversion = 1; /************************************************************************* Like MLPCreate0, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreate0(int nin, int nout, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreate0(nin, nout, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreate1, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreate1(int nin, int nhid, int nout, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreate1(nin, nhid, nout, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreate2, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreate2(int nin, int nhid1, int nhid2, int nout, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreate2(nin, nhid1, nhid2, nout, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreateB0, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreateb0(int nin, int nout, double b, double d, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreateb0(nin, nout, b, d, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreateB1, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreateb1(int nin, int nhid, int nout, double b, double d, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreateb1(nin, nhid, nout, b, d, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreateB2, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreateb2(int nin, int nhid1, int nhid2, int nout, double b, double d, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreateb2(nin, nhid1, nhid2, nout, b, d, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreateR0, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreater0(int nin, int nout, double a, double b, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreater0(nin, nout, a, b, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreateR1, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreater1(int nin, int nhid, int nout, double a, double b, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreater1(nin, nhid, nout, a, b, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreateR2, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreater2(int nin, int nhid1, int nhid2, int nout, double a, double b, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreater2(nin, nhid1, nhid2, nout, a, b, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreateC0, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreatec0(int nin, int nout, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreatec0(nin, nout, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreateC1, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreatec1(int nin, int nhid, int nout, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreatec1(nin, nhid, nout, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Like MLPCreateC2, but for ensembles. -- ALGLIB -- Copyright 18.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreatec2(int nin, int nhid1, int nhid2, int nout, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { mlpbase.multilayerperceptron net = new mlpbase.multilayerperceptron(); mlpbase.mlpcreatec2(nin, nhid1, nhid2, nout, net, _params); mlpecreatefromnetwork(net, ensemblesize, ensemble, _params); } /************************************************************************* Creates ensemble from network. Only network geometry is copied. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecreatefromnetwork(mlpbase.multilayerperceptron network, int ensemblesize, mlpensemble ensemble, alglib.xparams _params) { int i = 0; int ccount = 0; int wcount = 0; int i_ = 0; int i1_ = 0; alglib.ap.assert(ensemblesize>0, "MLPECreate: incorrect ensemble size!"); // // Copy network // mlpbase.mlpcopy(network, ensemble.network, _params); // // network properties // if( mlpbase.mlpissoftmax(network, _params) ) { ccount = mlpbase.mlpgetinputscount(ensemble.network, _params); } else { ccount = mlpbase.mlpgetinputscount(ensemble.network, _params)+mlpbase.mlpgetoutputscount(ensemble.network, _params); } wcount = mlpbase.mlpgetweightscount(ensemble.network, _params); ensemble.ensemblesize = ensemblesize; // // weights, means, sigmas // ensemble.weights = new double[ensemblesize*wcount]; ensemble.columnmeans = new double[ensemblesize*ccount]; ensemble.columnsigmas = new double[ensemblesize*ccount]; for(i=0; i<=ensemblesize*wcount-1; i++) { ensemble.weights[i] = math.randomreal()-0.5; } for(i=0; i<=ensemblesize-1; i++) { i1_ = (0) - (i*ccount); for(i_=i*ccount; i_<=(i+1)*ccount-1;i_++) { ensemble.columnmeans[i_] = network.columnmeans[i_+i1_]; } i1_ = (0) - (i*ccount); for(i_=i*ccount; i_<=(i+1)*ccount-1;i_++) { ensemble.columnsigmas[i_] = network.columnsigmas[i_+i1_]; } } // // temporaries, internal buffers // ensemble.y = new double[mlpbase.mlpgetoutputscount(ensemble.network, _params)]; } /************************************************************************* Copying of MLPEnsemble strucure INPUT PARAMETERS: Ensemble1 - original OUTPUT PARAMETERS: Ensemble2 - copy -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpecopy(mlpensemble ensemble1, mlpensemble ensemble2, alglib.xparams _params) { int ccount = 0; int wcount = 0; int i_ = 0; // // Unload info // if( mlpbase.mlpissoftmax(ensemble1.network, _params) ) { ccount = mlpbase.mlpgetinputscount(ensemble1.network, _params); } else { ccount = mlpbase.mlpgetinputscount(ensemble1.network, _params)+mlpbase.mlpgetoutputscount(ensemble1.network, _params); } wcount = mlpbase.mlpgetweightscount(ensemble1.network, _params); // // Allocate space // ensemble2.weights = new double[ensemble1.ensemblesize*wcount]; ensemble2.columnmeans = new double[ensemble1.ensemblesize*ccount]; ensemble2.columnsigmas = new double[ensemble1.ensemblesize*ccount]; ensemble2.y = new double[mlpbase.mlpgetoutputscount(ensemble1.network, _params)]; // // Copy // ensemble2.ensemblesize = ensemble1.ensemblesize; for(i_=0; i_<=ensemble1.ensemblesize*wcount-1;i_++) { ensemble2.weights[i_] = ensemble1.weights[i_]; } for(i_=0; i_<=ensemble1.ensemblesize*ccount-1;i_++) { ensemble2.columnmeans[i_] = ensemble1.columnmeans[i_]; } for(i_=0; i_<=ensemble1.ensemblesize*ccount-1;i_++) { ensemble2.columnsigmas[i_] = ensemble1.columnsigmas[i_]; } mlpbase.mlpcopy(ensemble1.network, ensemble2.network, _params); } /************************************************************************* Randomization of MLP ensemble -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlperandomize(mlpensemble ensemble, alglib.xparams _params) { int i = 0; int wcount = 0; wcount = mlpbase.mlpgetweightscount(ensemble.network, _params); for(i=0; i<=ensemble.ensemblesize*wcount-1; i++) { ensemble.weights[i] = math.randomreal()-0.5; } } /************************************************************************* Return ensemble properties (number of inputs and outputs). -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpeproperties(mlpensemble ensemble, ref int nin, ref int nout, alglib.xparams _params) { nin = 0; nout = 0; nin = mlpbase.mlpgetinputscount(ensemble.network, _params); nout = mlpbase.mlpgetoutputscount(ensemble.network, _params); } /************************************************************************* Return normalization type (whether ensemble is SOFTMAX-normalized or not). -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static bool mlpeissoftmax(mlpensemble ensemble, alglib.xparams _params) { bool result = new bool(); result = mlpbase.mlpissoftmax(ensemble.network, _params); return result; } /************************************************************************* Procesing INPUT PARAMETERS: Ensemble- neural networks ensemble X - input vector, array[0..NIn-1]. Y - (possibly) preallocated buffer; if size of Y is less than NOut, it will be reallocated. If it is large enough, it is NOT reallocated, so we can save some time on reallocation. OUTPUT PARAMETERS: Y - result. Regression estimate when solving regression task, vector of posterior probabilities for classification task. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpeprocess(mlpensemble ensemble, double[] x, ref double[] y, alglib.xparams _params) { int i = 0; int es = 0; int wc = 0; int cc = 0; double v = 0; int nout = 0; int i_ = 0; int i1_ = 0; if( alglib.ap.len(y)=0, "MLPEAllErrorsX: internal error"); if( datasettype==0 ) { for(i_=0; i_<=nin-1;i_++) { pbuf.x[i_] = densexy[srcidx,i_]; } } if( datasettype==1 ) { sparse.sparsegetrow(sparsexy, srcidx, ref pbuf.x, _params); } mlpeprocess(ensemble, pbuf.x, ref pbuf.y, _params); if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { if( datasettype==0 ) { pbuf.desiredy[0] = densexy[srcidx,nin]; } if( datasettype==1 ) { pbuf.desiredy[0] = sparse.sparseget(sparsexy, srcidx, nin, _params); } } else { if( datasettype==0 ) { i1_ = (nin) - (0); for(i_=0; i_<=nout-1;i_++) { pbuf.desiredy[i_] = densexy[srcidx,i_+i1_]; } } if( datasettype==1 ) { for(j=0; j<=nout-1; j++) { pbuf.desiredy[j] = sparse.sparseget(sparsexy, srcidx, nin+j, _params); } } } bdss.dserraccumulate(ref pbuf.tmp0, pbuf.y, pbuf.desiredy, _params); } bdss.dserrfinish(ref pbuf.tmp0, _params); rep.relclserror = pbuf.tmp0[0]; rep.avgce = pbuf.tmp0[1]/Math.Log(2); rep.rmserror = pbuf.tmp0[2]; rep.avgerror = pbuf.tmp0[3]; rep.avgrelerror = pbuf.tmp0[4]; alglib.smp.ae_shared_pool_recycle(buf, ref pbuf); } /************************************************************************* Calculation of all types of errors on dataset given by sparse matrix -- ALGLIB -- Copyright 10.09.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpeallerrorssparse(mlpensemble ensemble, sparse.sparsematrix xy, int npoints, ref double relcls, ref double avgce, ref double rms, ref double avg, ref double avgrel, alglib.xparams _params) { int i = 0; double[] buf = new double[0]; double[] workx = new double[0]; double[] y = new double[0]; double[] dy = new double[0]; int nin = 0; int nout = 0; int i_ = 0; int i1_ = 0; relcls = 0; avgce = 0; rms = 0; avg = 0; avgrel = 0; nin = mlpbase.mlpgetinputscount(ensemble.network, _params); nout = mlpbase.mlpgetoutputscount(ensemble.network, _params); if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { dy = new double[1]; bdss.dserrallocate(nout, ref buf, _params); } else { dy = new double[nout]; bdss.dserrallocate(-nout, ref buf, _params); } for(i=0; i<=npoints-1; i++) { sparse.sparsegetrow(xy, i, ref workx, _params); mlpeprocess(ensemble, workx, ref y, _params); if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { dy[0] = workx[nin]; } else { i1_ = (nin) - (0); for(i_=0; i_<=nout-1;i_++) { dy[i_] = workx[i_+i1_]; } } bdss.dserraccumulate(ref buf, y, dy, _params); } bdss.dserrfinish(ref buf, _params); relcls = buf[0]; avgce = buf[1]; rms = buf[2]; avg = buf[3]; avgrel = buf[4]; } /************************************************************************* Relative classification error on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: percent of incorrectly classified cases. Works both for classifier betwork and for regression networks which are used as classifiers. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlperelclserror(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; mlpbase.modelerrors rep = new mlpbase.modelerrors(); mlpeallerrorsx(ensemble, xy, ensemble.network.dummysxy, npoints, 0, ensemble.network.dummyidx, 0, npoints, 0, ensemble.network.buf, rep, _params); result = rep.relclserror; return result; } /************************************************************************* Average cross-entropy (in bits per element) on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: CrossEntropy/(NPoints*LN(2)). Zero if ensemble solves regression task. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpeavgce(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; mlpbase.modelerrors rep = new mlpbase.modelerrors(); mlpeallerrorsx(ensemble, xy, ensemble.network.dummysxy, npoints, 0, ensemble.network.dummyidx, 0, npoints, 0, ensemble.network.buf, rep, _params); result = rep.avgce; return result; } /************************************************************************* RMS error on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: root mean square error. Its meaning for regression task is obvious. As for classification task RMS error means error when estimating posterior probabilities. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpermserror(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; mlpbase.modelerrors rep = new mlpbase.modelerrors(); mlpeallerrorsx(ensemble, xy, ensemble.network.dummysxy, npoints, 0, ensemble.network.dummyidx, 0, npoints, 0, ensemble.network.buf, rep, _params); result = rep.rmserror; return result; } /************************************************************************* Average error on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: Its meaning for regression task is obvious. As for classification task it means average error when estimating posterior probabilities. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpeavgerror(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; mlpbase.modelerrors rep = new mlpbase.modelerrors(); mlpeallerrorsx(ensemble, xy, ensemble.network.dummysxy, npoints, 0, ensemble.network.dummyidx, 0, npoints, 0, ensemble.network.buf, rep, _params); result = rep.avgerror; return result; } /************************************************************************* Average relative error on the test set INPUT PARAMETERS: Ensemble- ensemble XY - test set NPoints - test set size RESULT: Its meaning for regression task is obvious. As for classification task it means average relative error when estimating posterior probabilities. -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static double mlpeavgrelerror(mlpensemble ensemble, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; mlpbase.modelerrors rep = new mlpbase.modelerrors(); mlpeallerrorsx(ensemble, xy, ensemble.network.dummysxy, npoints, 0, ensemble.network.dummyidx, 0, npoints, 0, ensemble.network.buf, rep, _params); result = rep.avgrelerror; return result; } /************************************************************************* Serializer: allocation -- ALGLIB -- Copyright 19.10.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpealloc(alglib.serializer s, mlpensemble ensemble, alglib.xparams _params) { s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); apserv.allocrealarray(s, ensemble.weights, -1, _params); apserv.allocrealarray(s, ensemble.columnmeans, -1, _params); apserv.allocrealarray(s, ensemble.columnsigmas, -1, _params); mlpbase.mlpalloc(s, ensemble.network, _params); } /************************************************************************* Serializer: serialization -- ALGLIB -- Copyright 14.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpeserialize(alglib.serializer s, mlpensemble ensemble, alglib.xparams _params) { s.serialize_int(scodes.getmlpeserializationcode(_params)); s.serialize_int(mlpefirstversion); s.serialize_int(ensemble.ensemblesize); apserv.serializerealarray(s, ensemble.weights, -1, _params); apserv.serializerealarray(s, ensemble.columnmeans, -1, _params); apserv.serializerealarray(s, ensemble.columnsigmas, -1, _params); mlpbase.mlpserialize(s, ensemble.network, _params); } /************************************************************************* Serializer: unserialization -- ALGLIB -- Copyright 14.03.2011 by Bochkanov Sergey *************************************************************************/ public static void mlpeunserialize(alglib.serializer s, mlpensemble ensemble, alglib.xparams _params) { int i0 = 0; int i1 = 0; // // check correctness of header // i0 = s.unserialize_int(); alglib.ap.assert(i0==scodes.getmlpeserializationcode(_params), "MLPEUnserialize: stream header corrupted"); i1 = s.unserialize_int(); alglib.ap.assert(i1==mlpefirstversion, "MLPEUnserialize: stream header corrupted"); // // Create network // ensemble.ensemblesize = s.unserialize_int(); apserv.unserializerealarray(s, ref ensemble.weights, _params); apserv.unserializerealarray(s, ref ensemble.columnmeans, _params); apserv.unserializerealarray(s, ref ensemble.columnsigmas, _params); mlpbase.mlpunserialize(s, ensemble.network, _params); // // Allocate termoraries // ensemble.y = new double[mlpbase.mlpgetoutputscount(ensemble.network, _params)]; } } public class mlptrain { /************************************************************************* Training report: * RelCLSError - fraction of misclassified cases. * AvgCE - acerage cross-entropy * RMSError - root-mean-square error * AvgError - average error * AvgRelError - average relative error * NGrad - number of gradient calculations * NHess - number of Hessian calculations * NCholesky - number of Cholesky decompositions NOTE 1: RelCLSError/AvgCE are zero on regression problems. NOTE 2: on classification problems RMSError/AvgError/AvgRelError contain errors in prediction of posterior probabilities *************************************************************************/ public class mlpreport : apobject { public double relclserror; public double avgce; public double rmserror; public double avgerror; public double avgrelerror; public int ngrad; public int nhess; public int ncholesky; public mlpreport() { init(); } public override void init() { } public override alglib.apobject make_copy() { mlpreport _result = new mlpreport(); _result.relclserror = relclserror; _result.avgce = avgce; _result.rmserror = rmserror; _result.avgerror = avgerror; _result.avgrelerror = avgrelerror; _result.ngrad = ngrad; _result.nhess = nhess; _result.ncholesky = ncholesky; return _result; } }; /************************************************************************* Cross-validation estimates of generalization error *************************************************************************/ public class mlpcvreport : apobject { public double relclserror; public double avgce; public double rmserror; public double avgerror; public double avgrelerror; public mlpcvreport() { init(); } public override void init() { } public override alglib.apobject make_copy() { mlpcvreport _result = new mlpcvreport(); _result.relclserror = relclserror; _result.avgce = avgce; _result.rmserror = rmserror; _result.avgerror = avgerror; _result.avgrelerror = avgrelerror; return _result; } }; /************************************************************************* Temporary data structures used by following functions: * TrainNetworkX * StartTrainingX * ContinueTrainingX This structure contains: * network being trained * fully initialized LBFGS optimizer (we have to call MinLBFGSRestartFrom() before using it; usually it is done by StartTrainingX() function). * additional temporary arrays This structure should be initialized with InitMLPTrnSession() call. *************************************************************************/ public class smlptrnsession : apobject { public double[] bestparameters; public double bestrmserror; public bool randomizenetwork; public mlpbase.multilayerperceptron network; public minlbfgs.minlbfgsstate optimizer; public minlbfgs.minlbfgsreport optimizerrep; public double[] wbuf0; public double[] wbuf1; public int[] allminibatches; public int[] currentminibatch; public rcommstate rstate; public int algoused; public int minibatchsize; public hqrnd.hqrndstate generator; public smlptrnsession() { init(); } public override void init() { bestparameters = new double[0]; network = new mlpbase.multilayerperceptron(); optimizer = new minlbfgs.minlbfgsstate(); optimizerrep = new minlbfgs.minlbfgsreport(); wbuf0 = new double[0]; wbuf1 = new double[0]; allminibatches = new int[0]; currentminibatch = new int[0]; rstate = new rcommstate(); generator = new hqrnd.hqrndstate(); } public override alglib.apobject make_copy() { smlptrnsession _result = new smlptrnsession(); _result.bestparameters = (double[])bestparameters.Clone(); _result.bestrmserror = bestrmserror; _result.randomizenetwork = randomizenetwork; _result.network = (mlpbase.multilayerperceptron)network.make_copy(); _result.optimizer = (minlbfgs.minlbfgsstate)optimizer.make_copy(); _result.optimizerrep = (minlbfgs.minlbfgsreport)optimizerrep.make_copy(); _result.wbuf0 = (double[])wbuf0.Clone(); _result.wbuf1 = (double[])wbuf1.Clone(); _result.allminibatches = (int[])allminibatches.Clone(); _result.currentminibatch = (int[])currentminibatch.Clone(); _result.rstate = (rcommstate)rstate.make_copy(); _result.algoused = algoused; _result.minibatchsize = minibatchsize; _result.generator = (hqrnd.hqrndstate)generator.make_copy(); return _result; } }; /************************************************************************* Temporary data structures used by following functions: * TrainEnsembleX This structure contains: * two arrays which can be used to store training and validation subsets * sessions for MLP training This structure should be initialized with InitMLPETrnSession() call. *************************************************************************/ public class mlpetrnsession : apobject { public int[] trnsubset; public int[] valsubset; public alglib.smp.shared_pool mlpsessions; public mlpreport mlprep; public mlpbase.multilayerperceptron network; public mlpetrnsession() { init(); } public override void init() { trnsubset = new int[0]; valsubset = new int[0]; mlpsessions = new alglib.smp.shared_pool(); mlprep = new mlpreport(); network = new mlpbase.multilayerperceptron(); } public override alglib.apobject make_copy() { mlpetrnsession _result = new mlpetrnsession(); _result.trnsubset = (int[])trnsubset.Clone(); _result.valsubset = (int[])valsubset.Clone(); _result.mlpsessions = (alglib.smp.shared_pool)mlpsessions.make_copy(); _result.mlprep = (mlpreport)mlprep.make_copy(); _result.network = (mlpbase.multilayerperceptron)network.make_copy(); return _result; } }; /************************************************************************* Trainer object for neural network. You should not try to access fields of this object directly - use ALGLIB functions to work with this object. *************************************************************************/ public class mlptrainer : apobject { public int nin; public int nout; public bool rcpar; public int lbfgsfactor; public double decay; public double wstep; public int maxits; public int datatype; public int npoints; public double[,] densexy; public sparse.sparsematrix sparsexy; public smlptrnsession session; public int ngradbatch; public int[] subset; public int subsetsize; public int[] valsubset; public int valsubsetsize; public int algokind; public int minibatchsize; public mlptrainer() { init(); } public override void init() { densexy = new double[0,0]; sparsexy = new sparse.sparsematrix(); session = new smlptrnsession(); subset = new int[0]; valsubset = new int[0]; } public override alglib.apobject make_copy() { mlptrainer _result = new mlptrainer(); _result.nin = nin; _result.nout = nout; _result.rcpar = rcpar; _result.lbfgsfactor = lbfgsfactor; _result.decay = decay; _result.wstep = wstep; _result.maxits = maxits; _result.datatype = datatype; _result.npoints = npoints; _result.densexy = (double[,])densexy.Clone(); _result.sparsexy = (sparse.sparsematrix)sparsexy.make_copy(); _result.session = (smlptrnsession)session.make_copy(); _result.ngradbatch = ngradbatch; _result.subset = (int[])subset.Clone(); _result.subsetsize = subsetsize; _result.valsubset = (int[])valsubset.Clone(); _result.valsubsetsize = valsubsetsize; _result.algokind = algokind; _result.minibatchsize = minibatchsize; return _result; } }; /************************************************************************* Internal record for parallelization function MLPFoldCV. *************************************************************************/ public class mlpparallelizationcv : apobject { public mlpbase.multilayerperceptron network; public mlpreport rep; public int[] subset; public int subsetsize; public double[] xyrow; public double[] y; public int ngrad; public alglib.smp.shared_pool trnpool; public mlpparallelizationcv() { init(); } public override void init() { network = new mlpbase.multilayerperceptron(); rep = new mlpreport(); subset = new int[0]; xyrow = new double[0]; y = new double[0]; trnpool = new alglib.smp.shared_pool(); } public override alglib.apobject make_copy() { mlpparallelizationcv _result = new mlpparallelizationcv(); _result.network = (mlpbase.multilayerperceptron)network.make_copy(); _result.rep = (mlpreport)rep.make_copy(); _result.subset = (int[])subset.Clone(); _result.subsetsize = subsetsize; _result.xyrow = (double[])xyrow.Clone(); _result.y = (double[])y.Clone(); _result.ngrad = ngrad; _result.trnpool = (alglib.smp.shared_pool)trnpool.make_copy(); return _result; } }; public const double mindecay = 0.001; public const int defaultlbfgsfactor = 6; /************************************************************************* Neural network training using modified Levenberg-Marquardt with exact Hessian calculation and regularization. Subroutine trains neural network with restarts from random positions. Algorithm is well suited for small and medium scale problems (hundreds of weights). INPUT PARAMETERS: Network - neural network with initialized geometry XY - training set NPoints - training set size Decay - weight decay constant, >=0.001 Decay term 'Decay*||Weights||^2' is added to error function. If you don't know what Decay to choose, use 0.001. Restarts - number of restarts from random position, >0. If you don't know what Restarts to choose, use 2. OUTPUT PARAMETERS: Network - trained neural network. Info - return code: * -9, if internal matrix inverse subroutine failed * -2, if there is a point with class number outside of [0..NOut-1]. * -1, if wrong parameters specified (NPoints<0, Restarts<1). * 2, if task has been solved. Rep - training report -- ALGLIB -- Copyright 10.03.2009 by Bochkanov Sergey *************************************************************************/ public static void mlptrainlm(mlpbase.multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, ref int info, mlpreport rep, alglib.xparams _params) { int nin = 0; int nout = 0; int wcount = 0; double lmsteptol = 0; int i = 0; int k = 0; double v = 0; double e = 0; double enew = 0; double xnorm2 = 0; double stepnorm = 0; double[] g = new double[0]; double[] d = new double[0]; double[,] h = new double[0,0]; double[,] hmod = new double[0,0]; double[,] z = new double[0,0]; bool spd = new bool(); double nu = 0; double lambdav = 0; double lambdaup = 0; double lambdadown = 0; minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport(); minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate(); double[] x = new double[0]; double[] y = new double[0]; double[] wbase = new double[0]; double[] wdir = new double[0]; double[] wt = new double[0]; double[] wx = new double[0]; int pass = 0; double[] wbest = new double[0]; double ebest = 0; int invinfo = 0; matinv.matinvreport invrep = new matinv.matinvreport(); int solverinfo = 0; directdensesolvers.densesolverreport solverrep = new directdensesolvers.densesolverreport(); int i_ = 0; info = 0; mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount, _params); lambdaup = 10; lambdadown = 0.3; lmsteptol = 0.001; // // Test for inputs // if( npoints<=0 || restarts<1 ) { info = -1; return; } if( mlpbase.mlpissoftmax(network, _params) ) { for(i=0; i<=npoints-1; i++) { if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout ) { info = -2; return; } } } decay = Math.Max(decay, mindecay); info = 2; // // Initialize data // rep.ngrad = 0; rep.nhess = 0; rep.ncholesky = 0; // // General case. // Prepare task and network. Allocate space. // mlpbase.mlpinitpreprocessor(network, xy, npoints, _params); g = new double[wcount-1+1]; h = new double[wcount-1+1, wcount-1+1]; hmod = new double[wcount-1+1, wcount-1+1]; wbase = new double[wcount-1+1]; wdir = new double[wcount-1+1]; wbest = new double[wcount-1+1]; wt = new double[wcount-1+1]; wx = new double[wcount-1+1]; ebest = math.maxrealnumber; // // Multiple passes // for(pass=1; pass<=restarts; pass++) { // // Initialize weights // mlpbase.mlprandomize(network, _params); // // First stage of the hybrid algorithm: LBFGS // for(i_=0; i_<=wcount-1;i_++) { wbase[i_] = network.weights[i_]; } minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 5), wbase, state, _params); minlbfgs.minlbfgssetcond(state, 0, 0, 0, Math.Max(25, wcount), _params); while( minlbfgs.minlbfgsiteration(state, _params) ) { // // gradient // for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = state.x[i_]; } mlpbase.mlpgradbatch(network, xy, npoints, ref state.f, ref state.g, _params); // // weight decay // v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } state.f = state.f+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { state.g[i_] = state.g[i_] + decay*network.weights[i_]; } // // next iteration // rep.ngrad = rep.ngrad+1; } minlbfgs.minlbfgsresults(state, ref wbase, internalrep, _params); for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = wbase[i_]; } // // Second stage of the hybrid algorithm: LM // // Initialize H with identity matrix, // G with gradient, // E with regularized error. // mlpbase.mlphessianbatch(network, xy, npoints, ref e, ref g, ref h, _params); v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } e = e+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { g[i_] = g[i_] + decay*network.weights[i_]; } for(k=0; k<=wcount-1; k++) { h[k,k] = h[k,k]+decay; } rep.nhess = rep.nhess+1; lambdav = 0.001; nu = 2; while( true ) { // // 1. HMod = H+lambda*I // 2. Try to solve (H+Lambda*I)*dx = -g. // Increase lambda if left part is not positive definite. // for(i=0; i<=wcount-1; i++) { for(i_=0; i_<=wcount-1;i_++) { hmod[i,i_] = h[i,i_]; } hmod[i,i] = hmod[i,i]+lambdav; } spd = trfac.spdmatrixcholesky(ref hmod, wcount, true, _params); rep.ncholesky = rep.ncholesky+1; if( !spd ) { lambdav = lambdav*lambdaup*nu; nu = nu*2; continue; } directdensesolvers.spdmatrixcholeskysolve(hmod, wcount, true, g, ref solverinfo, solverrep, ref wdir, _params); if( solverinfo<0 ) { lambdav = lambdav*lambdaup*nu; nu = nu*2; continue; } for(i_=0; i_<=wcount-1;i_++) { wdir[i_] = -1*wdir[i_]; } // // Lambda found. // 1. Save old w in WBase // 1. Test some stopping criterions // 2. If error(w+wdir)>error(w), increase lambda // for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = network.weights[i_] + wdir[i_]; } xnorm2 = 0.0; for(i_=0; i_<=wcount-1;i_++) { xnorm2 += network.weights[i_]*network.weights[i_]; } stepnorm = 0.0; for(i_=0; i_<=wcount-1;i_++) { stepnorm += wdir[i_]*wdir[i_]; } stepnorm = Math.Sqrt(stepnorm); enew = mlpbase.mlperror(network, xy, npoints, _params)+0.5*decay*xnorm2; if( (double)(stepnorm)<(double)(lmsteptol*(1+Math.Sqrt(xnorm2))) ) { break; } if( (double)(enew)>(double)(e) ) { lambdav = lambdav*lambdaup*nu; nu = nu*2; continue; } // // Optimize using inv(cholesky(H)) as preconditioner // matinv.rmatrixtrinverse(ref hmod, wcount, true, false, ref invinfo, invrep, _params); if( invinfo<=0 ) { // // if matrix can't be inverted then exit with errors // TODO: make WCount steps in direction suggested by HMod // info = -9; return; } for(i_=0; i_<=wcount-1;i_++) { wbase[i_] = network.weights[i_]; } for(i=0; i<=wcount-1; i++) { wt[i] = 0; } minlbfgs.minlbfgscreatex(wcount, wcount, wt, 1, 0.0, state, _params); minlbfgs.minlbfgssetcond(state, 0, 0, 0, 5, _params); while( minlbfgs.minlbfgsiteration(state, _params) ) { // // gradient // for(i=0; i<=wcount-1; i++) { v = 0.0; for(i_=i; i_<=wcount-1;i_++) { v += state.x[i_]*hmod[i,i_]; } network.weights[i] = wbase[i]+v; } mlpbase.mlpgradbatch(network, xy, npoints, ref state.f, ref g, _params); for(i=0; i<=wcount-1; i++) { state.g[i] = 0; } for(i=0; i<=wcount-1; i++) { v = g[i]; for(i_=i; i_<=wcount-1;i_++) { state.g[i_] = state.g[i_] + v*hmod[i,i_]; } } // // weight decay // grad(x'*x) = A'*(x0+A*t) // v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } state.f = state.f+0.5*decay*v; for(i=0; i<=wcount-1; i++) { v = decay*network.weights[i]; for(i_=i; i_<=wcount-1;i_++) { state.g[i_] = state.g[i_] + v*hmod[i,i_]; } } // // next iteration // rep.ngrad = rep.ngrad+1; } minlbfgs.minlbfgsresults(state, ref wt, internalrep, _params); // // Accept new position. // Calculate Hessian // for(i=0; i<=wcount-1; i++) { v = 0.0; for(i_=i; i_<=wcount-1;i_++) { v += wt[i_]*hmod[i,i_]; } network.weights[i] = wbase[i]+v; } mlpbase.mlphessianbatch(network, xy, npoints, ref e, ref g, ref h, _params); v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } e = e+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { g[i_] = g[i_] + decay*network.weights[i_]; } for(k=0; k<=wcount-1; k++) { h[k,k] = h[k,k]+decay; } rep.nhess = rep.nhess+1; // // Update lambda // lambdav = lambdav*lambdadown; nu = 2; } // // update WBest // v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } e = 0.5*decay*v+mlpbase.mlperror(network, xy, npoints, _params); if( (double)(e)<(double)(ebest) ) { ebest = e; for(i_=0; i_<=wcount-1;i_++) { wbest[i_] = network.weights[i_]; } } } // // copy WBest to output // for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = wbest[i_]; } } /************************************************************************* Neural network training using L-BFGS algorithm with regularization. Subroutine trains neural network with restarts from random positions. Algorithm is well suited for problems of any dimensionality (memory requirements and step complexity are linear by weights number). INPUT PARAMETERS: Network - neural network with initialized geometry XY - training set NPoints - training set size Decay - weight decay constant, >=0.001 Decay term 'Decay*||Weights||^2' is added to error function. If you don't know what Decay to choose, use 0.001. Restarts - number of restarts from random position, >0. If you don't know what Restarts to choose, use 2. WStep - stopping criterion. Algorithm stops if step size is less than WStep. Recommended value - 0.01. Zero step size means stopping after MaxIts iterations. MaxIts - stopping criterion. Algorithm stops after MaxIts iterations (NOT gradient calculations). Zero MaxIts means stopping when step is sufficiently small. OUTPUT PARAMETERS: Network - trained neural network. Info - return code: * -8, if both WStep=0 and MaxIts=0 * -2, if there is a point with class number outside of [0..NOut-1]. * -1, if wrong parameters specified (NPoints<0, Restarts<1). * 2, if task has been solved. Rep - training report -- ALGLIB -- Copyright 09.12.2007 by Bochkanov Sergey *************************************************************************/ public static void mlptrainlbfgs(mlpbase.multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, ref int info, mlpreport rep, alglib.xparams _params) { int i = 0; int pass = 0; int nin = 0; int nout = 0; int wcount = 0; double[] w = new double[0]; double[] wbest = new double[0]; double e = 0; double v = 0; double ebest = 0; minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport(); minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate(); int i_ = 0; info = 0; // // Test inputs, parse flags, read network geometry // if( (double)(wstep)==(double)(0) && maxits==0 ) { info = -8; return; } if( ((npoints<=0 || restarts<1) || (double)(wstep)<(double)(0)) || maxits<0 ) { info = -1; return; } mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount, _params); if( mlpbase.mlpissoftmax(network, _params) ) { for(i=0; i<=npoints-1; i++) { if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout ) { info = -2; return; } } } decay = Math.Max(decay, mindecay); info = 2; // // Prepare // mlpbase.mlpinitpreprocessor(network, xy, npoints, _params); w = new double[wcount-1+1]; wbest = new double[wcount-1+1]; ebest = math.maxrealnumber; // // Multiple starts // rep.ncholesky = 0; rep.nhess = 0; rep.ngrad = 0; for(pass=1; pass<=restarts; pass++) { // // Process // mlpbase.mlprandomize(network, _params); for(i_=0; i_<=wcount-1;i_++) { w[i_] = network.weights[i_]; } minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 10), w, state, _params); minlbfgs.minlbfgssetcond(state, 0.0, 0.0, wstep, maxits, _params); while( minlbfgs.minlbfgsiteration(state, _params) ) { for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = state.x[i_]; } mlpbase.mlpgradnbatch(network, xy, npoints, ref state.f, ref state.g, _params); v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } state.f = state.f+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { state.g[i_] = state.g[i_] + decay*network.weights[i_]; } rep.ngrad = rep.ngrad+1; } minlbfgs.minlbfgsresults(state, ref w, internalrep, _params); for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = w[i_]; } // // Compare with best // v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } e = mlpbase.mlperrorn(network, xy, npoints, _params)+0.5*decay*v; if( (double)(e)<(double)(ebest) ) { for(i_=0; i_<=wcount-1;i_++) { wbest[i_] = network.weights[i_]; } ebest = e; } } // // The best network // for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = wbest[i_]; } } /************************************************************************* Neural network training using early stopping (base algorithm - L-BFGS with regularization). INPUT PARAMETERS: Network - neural network with initialized geometry TrnXY - training set TrnSize - training set size, TrnSize>0 ValXY - validation set ValSize - validation set size, ValSize>0 Decay - weight decay constant, >=0.001 Decay term 'Decay*||Weights||^2' is added to error function. If you don't know what Decay to choose, use 0.001. Restarts - number of restarts, either: * strictly positive number - algorithm make specified number of restarts from random position. * -1, in which case algorithm makes exactly one run from the initial state of the network (no randomization). If you don't know what Restarts to choose, choose one one the following: * -1 (deterministic start) * +1 (one random restart) * +5 (moderate amount of random restarts) OUTPUT PARAMETERS: Network - trained neural network. Info - return code: * -2, if there is a point with class number outside of [0..NOut-1]. * -1, if wrong parameters specified (NPoints<0, Restarts<1, ...). * 2, task has been solved, stopping criterion met - sufficiently small step size. Not expected (we use EARLY stopping) but possible and not an error. * 6, task has been solved, stopping criterion met - increasing of validation set error. Rep - training report NOTE: Algorithm stops if validation set error increases for a long enough or step size is small enought (there are task where validation set may decrease for eternity). In any case solution returned corresponds to the minimum of validation set error. -- ALGLIB -- Copyright 10.03.2009 by Bochkanov Sergey *************************************************************************/ public static void mlptraines(mlpbase.multilayerperceptron network, double[,] trnxy, int trnsize, double[,] valxy, int valsize, double decay, int restarts, ref int info, mlpreport rep, alglib.xparams _params) { int i = 0; int pass = 0; int nin = 0; int nout = 0; int wcount = 0; double[] w = new double[0]; double[] wbest = new double[0]; double e = 0; double v = 0; double ebest = 0; double[] wfinal = new double[0]; double efinal = 0; int itcnt = 0; int itbest = 0; minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport(); minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate(); double wstep = 0; bool needrandomization = new bool(); int i_ = 0; info = 0; wstep = 0.001; // // Test inputs, parse flags, read network geometry // if( ((trnsize<=0 || valsize<=0) || (restarts<1 && restarts!=-1)) || (double)(decay)<(double)(0) ) { info = -1; return; } if( restarts==-1 ) { needrandomization = false; restarts = 1; } else { needrandomization = true; } mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount, _params); if( mlpbase.mlpissoftmax(network, _params) ) { for(i=0; i<=trnsize-1; i++) { if( (int)Math.Round(trnxy[i,nin])<0 || (int)Math.Round(trnxy[i,nin])>=nout ) { info = -2; return; } } for(i=0; i<=valsize-1; i++) { if( (int)Math.Round(valxy[i,nin])<0 || (int)Math.Round(valxy[i,nin])>=nout ) { info = -2; return; } } } info = 2; // // Prepare // mlpbase.mlpinitpreprocessor(network, trnxy, trnsize, _params); w = new double[wcount-1+1]; wbest = new double[wcount-1+1]; wfinal = new double[wcount-1+1]; efinal = math.maxrealnumber; for(i=0; i<=wcount-1; i++) { wfinal[i] = 0; } // // Multiple starts // rep.ncholesky = 0; rep.nhess = 0; rep.ngrad = 0; for(pass=1; pass<=restarts; pass++) { // // Process // if( needrandomization ) { mlpbase.mlprandomize(network, _params); } ebest = mlpbase.mlperror(network, valxy, valsize, _params); for(i_=0; i_<=wcount-1;i_++) { wbest[i_] = network.weights[i_]; } itbest = 0; itcnt = 0; for(i_=0; i_<=wcount-1;i_++) { w[i_] = network.weights[i_]; } minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 10), w, state, _params); minlbfgs.minlbfgssetcond(state, 0.0, 0.0, wstep, 0, _params); minlbfgs.minlbfgssetxrep(state, true, _params); while( minlbfgs.minlbfgsiteration(state, _params) ) { // // Calculate gradient // if( state.needfg ) { for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = state.x[i_]; } mlpbase.mlpgradnbatch(network, trnxy, trnsize, ref state.f, ref state.g, _params); v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += network.weights[i_]*network.weights[i_]; } state.f = state.f+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { state.g[i_] = state.g[i_] + decay*network.weights[i_]; } rep.ngrad = rep.ngrad+1; } // // Validation set // if( state.xupdated ) { for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = state.x[i_]; } e = mlpbase.mlperror(network, valxy, valsize, _params); if( (double)(e)<(double)(ebest) ) { ebest = e; for(i_=0; i_<=wcount-1;i_++) { wbest[i_] = network.weights[i_]; } itbest = itcnt; } if( itcnt>30 && (double)(itcnt)>(double)(1.5*itbest) ) { info = 6; break; } itcnt = itcnt+1; } } minlbfgs.minlbfgsresults(state, ref w, internalrep, _params); // // Compare with final answer // if( (double)(ebest)<(double)(efinal) ) { for(i_=0; i_<=wcount-1;i_++) { wfinal[i_] = wbest[i_]; } efinal = ebest; } } // // The best network // for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = wfinal[i_]; } } /************************************************************************* Cross-validation estimate of generalization error. Base algorithm - L-BFGS. INPUT PARAMETERS: Network - neural network with initialized geometry. Network is not changed during cross-validation - it is used only as a representative of its architecture. XY - training set. SSize - training set size Decay - weight decay, same as in MLPTrainLBFGS Restarts - number of restarts, >0. restarts are counted for each partition separately, so total number of restarts will be Restarts*FoldsCount. WStep - stopping criterion, same as in MLPTrainLBFGS MaxIts - stopping criterion, same as in MLPTrainLBFGS FoldsCount - number of folds in k-fold cross-validation, 2<=FoldsCount<=SSize. recommended value: 10. OUTPUT PARAMETERS: Info - return code, same as in MLPTrainLBFGS Rep - report, same as in MLPTrainLM/MLPTrainLBFGS CVRep - generalization error estimates -- ALGLIB -- Copyright 09.12.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpkfoldcvlbfgs(mlpbase.multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, int foldscount, ref int info, mlpreport rep, mlpcvreport cvrep, alglib.xparams _params) { info = 0; mlpkfoldcvgeneral(network, xy, npoints, decay, restarts, foldscount, false, wstep, maxits, ref info, rep, cvrep, _params); } /************************************************************************* Cross-validation estimate of generalization error. Base algorithm - Levenberg-Marquardt. INPUT PARAMETERS: Network - neural network with initialized geometry. Network is not changed during cross-validation - it is used only as a representative of its architecture. XY - training set. SSize - training set size Decay - weight decay, same as in MLPTrainLBFGS Restarts - number of restarts, >0. restarts are counted for each partition separately, so total number of restarts will be Restarts*FoldsCount. FoldsCount - number of folds in k-fold cross-validation, 2<=FoldsCount<=SSize. recommended value: 10. OUTPUT PARAMETERS: Info - return code, same as in MLPTrainLBFGS Rep - report, same as in MLPTrainLM/MLPTrainLBFGS CVRep - generalization error estimates -- ALGLIB -- Copyright 09.12.2007 by Bochkanov Sergey *************************************************************************/ public static void mlpkfoldcvlm(mlpbase.multilayerperceptron network, double[,] xy, int npoints, double decay, int restarts, int foldscount, ref int info, mlpreport rep, mlpcvreport cvrep, alglib.xparams _params) { info = 0; mlpkfoldcvgeneral(network, xy, npoints, decay, restarts, foldscount, true, 0.0, 0, ref info, rep, cvrep, _params); } /************************************************************************* This function estimates generalization error using cross-validation on the current dataset with current training settings. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - trainer object Network - neural network. It must have same number of inputs and output/classes as was specified during creation of the trainer object. Network is not changed during cross- validation and is not trained - it is used only as representative of its architecture. I.e., we estimate generalization properties of ARCHITECTURE, not some specific network. NRestarts - number of restarts, >=0: * NRestarts>0 means that for each cross-validation round specified number of random restarts is performed, with best network being chosen after training. * NRestarts=0 is same as NRestarts=1 FoldsCount - number of folds in k-fold cross-validation: * 2<=FoldsCount<=size of dataset * recommended value: 10. * values larger than dataset size will be silently truncated down to dataset size OUTPUT PARAMETERS: Rep - structure which contains cross-validation estimates: * Rep.RelCLSError - fraction of misclassified cases. * Rep.AvgCE - acerage cross-entropy * Rep.RMSError - root-mean-square error * Rep.AvgError - average error * Rep.AvgRelError - average relative error NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(), or subset with only one point was given, zeros are returned as estimates. NOTE: this method performs FoldsCount cross-validation rounds, each one with NRestarts random starts. Thus, FoldsCount*NRestarts networks are trained in total. NOTE: Rep.RelCLSError/Rep.AvgCE are zero on regression problems. NOTE: on classification problems Rep.RMSError/Rep.AvgError/Rep.AvgRelError contain errors in prediction of posterior probabilities. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpkfoldcv(mlptrainer s, mlpbase.multilayerperceptron network, int nrestarts, int foldscount, mlpreport rep, alglib.xparams _params) { alglib.smp.shared_pool pooldatacv = new alglib.smp.shared_pool(); mlpparallelizationcv datacv = new mlpparallelizationcv(); mlpparallelizationcv sdatacv = null; double[,] cvy = new double[0,0]; int[] folds = new int[0]; double[] buf = new double[0]; double[] dy = new double[0]; int nin = 0; int nout = 0; int wcount = 0; int rowsize = 0; int ntype = 0; int ttype = 0; int i = 0; int j = 0; int k = 0; hqrnd.hqrndstate rs = new hqrnd.hqrndstate(); int i_ = 0; int i1_ = 0; if( !mlpbase.mlpissoftmax(network, _params) ) { ntype = 0; } else { ntype = 1; } if( s.rcpar ) { ttype = 0; } else { ttype = 1; } alglib.ap.assert(ntype==ttype, "MLPKFoldCV: type of input network is not similar to network type in trainer object"); alglib.ap.assert(s.npoints>=0, "MLPKFoldCV: possible trainer S is not initialized(S.NPoints<0)"); mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount, _params); alglib.ap.assert(s.nin==nin, "MLPKFoldCV: number of inputs in trainer is not equal to number of inputs in network"); alglib.ap.assert(s.nout==nout, "MLPKFoldCV: number of outputs in trainer is not equal to number of outputs in network"); alglib.ap.assert(nrestarts>=0, "MLPKFoldCV: NRestarts<0"); alglib.ap.assert(foldscount>=2, "MLPKFoldCV: FoldsCount<2"); if( foldscount>s.npoints ) { foldscount = s.npoints; } rep.relclserror = 0; rep.avgce = 0; rep.rmserror = 0; rep.avgerror = 0; rep.avgrelerror = 0; hqrnd.hqrndrandomize(rs, _params); rep.ngrad = 0; rep.nhess = 0; rep.ncholesky = 0; if( s.npoints==0 || s.npoints==1 ) { return; } // // Read network geometry, test parameters // if( s.rcpar ) { rowsize = nin+nout; dy = new double[nout]; bdss.dserrallocate(-nout, ref buf, _params); } else { rowsize = nin+1; dy = new double[1]; bdss.dserrallocate(nout, ref buf, _params); } // // Folds // folds = new int[s.npoints]; for(i=0; i<=s.npoints-1; i++) { folds[i] = i*foldscount/s.npoints; } for(i=0; i<=s.npoints-2; i++) { j = i+hqrnd.hqrnduniformi(rs, s.npoints-i, _params); if( j!=i ) { k = folds[i]; folds[i] = folds[j]; folds[j] = k; } } cvy = new double[s.npoints, nout]; // // Initialize SEED-value for shared pool // datacv.ngrad = 0; mlpbase.mlpcopy(network, datacv.network, _params); datacv.subset = new int[s.npoints]; datacv.xyrow = new double[rowsize]; datacv.y = new double[nout]; // // Create shared pool // alglib.smp.ae_shared_pool_set_seed(pooldatacv, datacv); // // Parallelization // mthreadcv(s, rowsize, nrestarts, folds, 0, foldscount, cvy, pooldatacv, wcount, _params); // // Calculate value for NGrad // alglib.smp.ae_shared_pool_first_recycled(pooldatacv, ref sdatacv); while( sdatacv!=null ) { rep.ngrad = rep.ngrad+sdatacv.ngrad; alglib.smp.ae_shared_pool_next_recycled(pooldatacv, ref sdatacv); } // // Connect of results and calculate cross-validation error // for(i=0; i<=s.npoints-1; i++) { if( s.datatype==0 ) { for(i_=0; i_<=rowsize-1;i_++) { datacv.xyrow[i_] = s.densexy[i,i_]; } } if( s.datatype==1 ) { sparse.sparsegetrow(s.sparsexy, i, ref datacv.xyrow, _params); } for(i_=0; i_<=nout-1;i_++) { datacv.y[i_] = cvy[i,i_]; } if( s.rcpar ) { i1_ = (nin) - (0); for(i_=0; i_<=nout-1;i_++) { dy[i_] = datacv.xyrow[i_+i1_]; } } else { dy[0] = datacv.xyrow[nin]; } bdss.dserraccumulate(ref buf, datacv.y, dy, _params); } bdss.dserrfinish(ref buf, _params); rep.relclserror = buf[0]; rep.avgce = buf[1]; rep.rmserror = buf[2]; rep.avgerror = buf[3]; rep.avgrelerror = buf[4]; } /************************************************************************* Creation of the network trainer object for regression networks INPUT PARAMETERS: NIn - number of inputs, NIn>=1 NOut - number of outputs, NOut>=1 OUTPUT PARAMETERS: S - neural network trainer object. This structure can be used to train any regression network with NIn inputs and NOut outputs. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatetrainer(int nin, int nout, mlptrainer s, alglib.xparams _params) { alglib.ap.assert(nin>=1, "MLPCreateTrainer: NIn<1."); alglib.ap.assert(nout>=1, "MLPCreateTrainer: NOut<1."); s.nin = nin; s.nout = nout; s.rcpar = true; s.lbfgsfactor = defaultlbfgsfactor; s.decay = 1.0E-6; mlpsetcond(s, 0, 0, _params); s.datatype = 0; s.npoints = 0; mlpsetalgobatch(s, _params); } /************************************************************************* Creation of the network trainer object for classification networks INPUT PARAMETERS: NIn - number of inputs, NIn>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: S - neural network trainer object. This structure can be used to train any classification network with NIn inputs and NOut outputs. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpcreatetrainercls(int nin, int nclasses, mlptrainer s, alglib.xparams _params) { alglib.ap.assert(nin>=1, "MLPCreateTrainerCls: NIn<1."); alglib.ap.assert(nclasses>=2, "MLPCreateTrainerCls: NClasses<2."); s.nin = nin; s.nout = nclasses; s.rcpar = false; s.lbfgsfactor = defaultlbfgsfactor; s.decay = 1.0E-6; mlpsetcond(s, 0, 0, _params); s.datatype = 0; s.npoints = 0; mlpsetalgobatch(s, _params); } /************************************************************************* This function sets "current dataset" of the trainer object to one passed by user. INPUT PARAMETERS: S - trainer object XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. NPoints - points count, >=0. DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following datasetformat is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetdataset(mlptrainer s, double[,] xy, int npoints, alglib.xparams _params) { int ndim = 0; int i = 0; int j = 0; alglib.ap.assert(s.nin>=1, "MLPSetDataset: possible parameter S is not initialized or spoiled(S.NIn<=0)."); alglib.ap.assert(npoints>=0, "MLPSetDataset: NPoint<0"); alglib.ap.assert(npoints<=alglib.ap.rows(xy), "MLPSetDataset: invalid size of matrix XY(NPoint more then rows of matrix XY)"); s.datatype = 0; s.npoints = npoints; if( npoints==0 ) { return; } if( s.rcpar ) { alglib.ap.assert(s.nout>=1, "MLPSetDataset: possible parameter S is not initialized or is spoiled(NOut<1 for regression)."); ndim = s.nin+s.nout; alglib.ap.assert(ndim<=alglib.ap.cols(xy), "MLPSetDataset: invalid size of matrix XY(too few columns in matrix XY)."); alglib.ap.assert(apserv.apservisfinitematrix(xy, npoints, ndim, _params), "MLPSetDataset: parameter XY contains Infinite or NaN."); } else { alglib.ap.assert(s.nout>=2, "MLPSetDataset: possible parameter S is not initialized or is spoiled(NClasses<2 for classifier)."); ndim = s.nin+1; alglib.ap.assert(ndim<=alglib.ap.cols(xy), "MLPSetDataset: invalid size of matrix XY(too few columns in matrix XY)."); alglib.ap.assert(apserv.apservisfinitematrix(xy, npoints, ndim, _params), "MLPSetDataset: parameter XY contains Infinite or NaN."); for(i=0; i<=npoints-1; i++) { alglib.ap.assert((int)Math.Round(xy[i,s.nin])>=0 && (int)Math.Round(xy[i,s.nin])=NClasses)."); } } apserv.rmatrixsetlengthatleast(ref s.densexy, npoints, ndim, _params); for(i=0; i<=npoints-1; i++) { for(j=0; j<=ndim-1; j++) { s.densexy[i,j] = xy[i,j]; } } } /************************************************************************* This function sets "current dataset" of the trainer object to one passed by user (sparse matrix is used to store dataset). INPUT PARAMETERS: S - trainer object XY - training set, see below for information on the training set format. This function checks correctness of the dataset (no NANs/INFs, class numbers are correct) and throws exception when incorrect dataset is passed. Any sparse storage format can be used: Hash-table, CRS... NPoints - points count, >=0 DATASET FORMAT: This function uses two different dataset formats - one for regression networks, another one for classification networks. For regression networks with NIn inputs and NOut outputs following dataset format is used: * dataset is given by NPoints*(NIn+NOut) matrix * each row corresponds to one example * first NIn columns are inputs, next NOut columns are outputs For classification networks with NIn inputs and NClasses clases following datasetformat is used: * dataset is given by NPoints*(NIn+1) matrix * each row corresponds to one example * first NIn columns are inputs, last column stores class number (from 0 to NClasses-1). -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetsparsedataset(mlptrainer s, sparse.sparsematrix xy, int npoints, alglib.xparams _params) { double v = 0; int t0 = 0; int t1 = 0; int i = 0; int j = 0; // // Check correctness of the data // alglib.ap.assert(s.nin>0, "MLPSetSparseDataset: possible parameter S is not initialized or spoiled(S.NIn<=0)."); alglib.ap.assert(npoints>=0, "MLPSetSparseDataset: NPoint<0"); alglib.ap.assert(npoints<=sparse.sparsegetnrows(xy, _params), "MLPSetSparseDataset: invalid size of sparse matrix XY(NPoint more then rows of matrix XY)"); if( npoints>0 ) { t0 = 0; t1 = 0; if( s.rcpar ) { alglib.ap.assert(s.nout>=1, "MLPSetSparseDataset: possible parameter S is not initialized or is spoiled(NOut<1 for regression)."); alglib.ap.assert(s.nin+s.nout<=sparse.sparsegetncols(xy, _params), "MLPSetSparseDataset: invalid size of sparse matrix XY(too few columns in sparse matrix XY)."); while( sparse.sparseenumerate(xy, ref t0, ref t1, ref i, ref j, ref v, _params) ) { if( i=2, "MLPSetSparseDataset: possible parameter S is not initialized or is spoiled(NClasses<2 for classifier)."); alglib.ap.assert(s.nin+1<=sparse.sparsegetncols(xy, _params), "MLPSetSparseDataset: invalid size of sparse matrix XY(too few columns in sparse matrix XY)."); while( sparse.sparseenumerate(xy, ref t0, ref t1, ref i, ref j, ref v, _params) ) { if( i=0) && (int)Math.Round(v)=NClasses)."); } } } } } // // Set dataset // s.datatype = 1; s.npoints = npoints; sparse.sparsecopytocrs(xy, s.sparsexy, _params); } /************************************************************************* This function sets weight decay coefficient which is used for training. INPUT PARAMETERS: S - trainer object Decay - weight decay coefficient, >=0. Weight decay term 'Decay*||Weights||^2' is added to error function. If you don't know what Decay to choose, use 1.0E-3. Weight decay can be set to zero, in this case network is trained without weight decay. NOTE: by default network uses some small nonzero value for weight decay. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetdecay(mlptrainer s, double decay, alglib.xparams _params) { alglib.ap.assert(math.isfinite(decay), "MLPSetDecay: parameter Decay contains Infinite or NaN."); alglib.ap.assert((double)(decay)>=(double)(0), "MLPSetDecay: Decay<0."); s.decay = decay; } /************************************************************************* This function sets stopping criteria for the optimizer. INPUT PARAMETERS: S - trainer object WStep - stopping criterion. Algorithm stops if step size is less than WStep. Recommended value - 0.01. Zero step size means stopping after MaxIts iterations. WStep>=0. MaxIts - stopping criterion. Algorithm stops after MaxIts epochs (full passes over entire dataset). Zero MaxIts means stopping when step is sufficiently small. MaxIts>=0. NOTE: by default, WStep=0.005 and MaxIts=0 are used. These values are also used when MLPSetCond() is called with WStep=0 and MaxIts=0. NOTE: these stopping criteria are used for all kinds of neural training - from "conventional" networks to early stopping ensembles. When used for "conventional" networks, they are used as the only stopping criteria. When combined with early stopping, they used as ADDITIONAL stopping criteria which can terminate early stopping algorithm. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetcond(mlptrainer s, double wstep, int maxits, alglib.xparams _params) { alglib.ap.assert(math.isfinite(wstep), "MLPSetCond: parameter WStep contains Infinite or NaN."); alglib.ap.assert((double)(wstep)>=(double)(0), "MLPSetCond: WStep<0."); alglib.ap.assert(maxits>=0, "MLPSetCond: MaxIts<0."); if( (double)(wstep)!=(double)(0) || maxits!=0 ) { s.wstep = wstep; s.maxits = maxits; } else { s.wstep = 0.005; s.maxits = 0; } } /************************************************************************* This function sets training algorithm: batch training using L-BFGS will be used. This algorithm: * the most robust for small-scale problems, but may be too slow for large scale ones. * perfoms full pass through the dataset before performing step * uses conditions specified by MLPSetCond() for stopping * is default one used by trainer object INPUT PARAMETERS: S - trainer object -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpsetalgobatch(mlptrainer s, alglib.xparams _params) { s.algokind = 0; } /************************************************************************* This function trains neural network passed to this function, using current dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset()) and current training settings. Training from NRestarts random starting positions is performed, best network is chosen. Training is performed using current training algorithm. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - trainer object Network - neural network. It must have same number of inputs and output/classes as was specified during creation of the trainer object. NRestarts - number of restarts, >=0: * NRestarts>0 means that specified number of random restarts are performed, best network is chosen after training * NRestarts=0 means that current state of the network is used for training. OUTPUT PARAMETERS: Network - trained network NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(), network is filled by zero values. Same behavior for functions MLPStartTraining and MLPContinueTraining. NOTE: this method uses sum-of-squares error function for training. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlptrainnetwork(mlptrainer s, mlpbase.multilayerperceptron network, int nrestarts, mlpreport rep, alglib.xparams _params) { int nin = 0; int nout = 0; int wcount = 0; int ntype = 0; int ttype = 0; alglib.smp.shared_pool trnpool = new alglib.smp.shared_pool(); alglib.ap.assert(s.npoints>=0, "MLPTrainNetwork: parameter S is not initialized or is spoiled(S.NPoints<0)"); if( !mlpbase.mlpissoftmax(network, _params) ) { ntype = 0; } else { ntype = 1; } if( s.rcpar ) { ttype = 0; } else { ttype = 1; } alglib.ap.assert(ntype==ttype, "MLPTrainNetwork: type of input network is not similar to network type in trainer object"); mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount, _params); alglib.ap.assert(s.nin==nin, "MLPTrainNetwork: number of inputs in trainer is not equal to number of inputs in network"); alglib.ap.assert(s.nout==nout, "MLPTrainNetwork: number of outputs in trainer is not equal to number of outputs in network"); alglib.ap.assert(nrestarts>=0, "MLPTrainNetwork: NRestarts<0."); // // Train // mlptrainnetworkx(s, nrestarts, -1, s.subset, -1, s.subset, 0, network, rep, true, trnpool, _params); } /************************************************************************* IMPORTANT: this is an "expert" version of the MLPTrain() function. We do not recommend you to use it unless you are pretty sure that you need ability to monitor training progress. This function performs step-by-step training of the neural network. Here "step-by-step" means that training starts with MLPStartTraining() call, and then user subsequently calls MLPContinueTraining() to perform one more iteration of the training. After call to this function trainer object remembers network and is ready to train it. However, no training is performed until first call to MLPContinueTraining() function. Subsequent calls to MLPContinueTraining() will advance training progress one iteration further. EXAMPLE: > > ...initialize network and trainer object.... > > MLPStartTraining(Trainer, Network, True) > while MLPContinueTraining(Trainer, Network) do > ...visualize training progress... > INPUT PARAMETERS: S - trainer object Network - neural network. It must have same number of inputs and output/classes as was specified during creation of the trainer object. RandomStart - randomize network before training or not: * True means that network is randomized and its initial state (one which was passed to the trainer object) is lost. * False means that training is started from the current state of the network OUTPUT PARAMETERS: Network - neural network which is ready to training (weights are initialized, preprocessor is initialized using current training set) NOTE: this method uses sum-of-squares error function for training. NOTE: it is expected that trainer object settings are NOT changed during step-by-step training, i.e. no one changes stopping criteria or training set during training. It is possible and there is no defense against such actions, but algorithm behavior in such cases is undefined and can be unpredictable. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static void mlpstarttraining(mlptrainer s, mlpbase.multilayerperceptron network, bool randomstart, alglib.xparams _params) { int nin = 0; int nout = 0; int wcount = 0; int ntype = 0; int ttype = 0; alglib.ap.assert(s.npoints>=0, "MLPStartTraining: parameter S is not initialized or is spoiled(S.NPoints<0)"); if( !mlpbase.mlpissoftmax(network, _params) ) { ntype = 0; } else { ntype = 1; } if( s.rcpar ) { ttype = 0; } else { ttype = 1; } alglib.ap.assert(ntype==ttype, "MLPStartTraining: type of input network is not similar to network type in trainer object"); mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount, _params); alglib.ap.assert(s.nin==nin, "MLPStartTraining: number of inputs in trainer is not equal to number of inputs in the network."); alglib.ap.assert(s.nout==nout, "MLPStartTraining: number of outputs in trainer is not equal to number of outputs in the network."); // // Initialize temporaries // initmlptrnsession(network, randomstart, s, s.session, _params); // // Train network // mlpstarttrainingx(s, randomstart, -1, s.subset, -1, s.session, _params); // // Update network // mlpbase.mlpcopytunableparameters(s.session.network, network, _params); } /************************************************************************* IMPORTANT: this is an "expert" version of the MLPTrain() function. We do not recommend you to use it unless you are pretty sure that you need ability to monitor training progress. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. This function performs step-by-step training of the neural network. Here "step-by-step" means that training starts with MLPStartTraining() call, and then user subsequently calls MLPContinueTraining() to perform one more iteration of the training. This function performs one more iteration of the training and returns either True (training continues) or False (training stopped). In case True was returned, Network weights are updated according to the current state of the optimization progress. In case False was returned, no additional updates is performed (previous update of the network weights moved us to the final point, and no additional updates is needed). EXAMPLE: > > [initialize network and trainer object] > > MLPStartTraining(Trainer, Network, True) > while MLPContinueTraining(Trainer, Network) do > [visualize training progress] > INPUT PARAMETERS: S - trainer object Network - neural network structure, which is used to store current state of the training process. OUTPUT PARAMETERS: Network - weights of the neural network are rewritten by the current approximation. NOTE: this method uses sum-of-squares error function for training. NOTE: it is expected that trainer object settings are NOT changed during step-by-step training, i.e. no one changes stopping criteria or training set during training. It is possible and there is no defense against such actions, but algorithm behavior in such cases is undefined and can be unpredictable. NOTE: It is expected that Network is the same one which was passed to MLPStartTraining() function. However, THIS function checks only following: * that number of network inputs is consistent with trainer object settings * that number of network outputs/classes is consistent with trainer object settings * that number of network weights is the same as number of weights in the network passed to MLPStartTraining() function Exception is thrown when these conditions are violated. It is also expected that you do not change state of the network on your own - the only party who has right to change network during its training is a trainer object. Any attempt to interfere with trainer may lead to unpredictable results. -- ALGLIB -- Copyright 23.07.2012 by Bochkanov Sergey *************************************************************************/ public static bool mlpcontinuetraining(mlptrainer s, mlpbase.multilayerperceptron network, alglib.xparams _params) { bool result = new bool(); int nin = 0; int nout = 0; int wcount = 0; int ntype = 0; int ttype = 0; int i_ = 0; alglib.ap.assert(s.npoints>=0, "MLPContinueTraining: parameter S is not initialized or is spoiled(S.NPoints<0)"); if( s.rcpar ) { ttype = 0; } else { ttype = 1; } if( !mlpbase.mlpissoftmax(network, _params) ) { ntype = 0; } else { ntype = 1; } alglib.ap.assert(ntype==ttype, "MLPContinueTraining: type of input network is not similar to network type in trainer object."); mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount, _params); alglib.ap.assert(s.nin==nin, "MLPContinueTraining: number of inputs in trainer is not equal to number of inputs in the network."); alglib.ap.assert(s.nout==nout, "MLPContinueTraining: number of outputs in trainer is not equal to number of outputs in the network."); result = mlpcontinuetrainingx(s, s.subset, -1, ref s.ngradbatch, s.session, _params); if( result ) { for(i_=0; i_<=wcount-1;i_++) { network.weights[i_] = s.session.network.weights[i_]; } } return result; } /************************************************************************* Training neural networks ensemble using bootstrap aggregating (bagging). Modified Levenberg-Marquardt algorithm is used as base training method. INPUT PARAMETERS: Ensemble - model with initialized geometry XY - training set NPoints - training set size Decay - weight decay coefficient, >=0.001 Restarts - restarts, >0. OUTPUT PARAMETERS: Ensemble - trained model Info - return code: * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, Restarts<1). * 2, if task has been solved. Rep - training report. OOBErrors - out-of-bag generalization error estimate -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpebagginglm(mlpe.mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, ref int info, mlpreport rep, mlpcvreport ooberrors, alglib.xparams _params) { info = 0; mlpebagginginternal(ensemble, xy, npoints, decay, restarts, 0.0, 0, true, ref info, rep, ooberrors, _params); } /************************************************************************* Training neural networks ensemble using bootstrap aggregating (bagging). L-BFGS algorithm is used as base training method. INPUT PARAMETERS: Ensemble - model with initialized geometry XY - training set NPoints - training set size Decay - weight decay coefficient, >=0.001 Restarts - restarts, >0. WStep - stopping criterion, same as in MLPTrainLBFGS MaxIts - stopping criterion, same as in MLPTrainLBFGS OUTPUT PARAMETERS: Ensemble - trained model Info - return code: * -8, if both WStep=0 and MaxIts=0 * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, Restarts<1). * 2, if task has been solved. Rep - training report. OOBErrors - out-of-bag generalization error estimate -- ALGLIB -- Copyright 17.02.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpebagginglbfgs(mlpe.mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, ref int info, mlpreport rep, mlpcvreport ooberrors, alglib.xparams _params) { info = 0; mlpebagginginternal(ensemble, xy, npoints, decay, restarts, wstep, maxits, false, ref info, rep, ooberrors, _params); } /************************************************************************* Training neural networks ensemble using early stopping. INPUT PARAMETERS: Ensemble - model with initialized geometry XY - training set NPoints - training set size Decay - weight decay coefficient, >=0.001 Restarts - restarts, >0. OUTPUT PARAMETERS: Ensemble - trained model Info - return code: * -2, if there is a point with class number outside of [0..NClasses-1]. * -1, if incorrect parameters was passed (NPoints<0, Restarts<1). * 6, if task has been solved. Rep - training report. OOBErrors - out-of-bag generalization error estimate -- ALGLIB -- Copyright 10.03.2009 by Bochkanov Sergey *************************************************************************/ public static void mlpetraines(mlpe.mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, ref int info, mlpreport rep, alglib.xparams _params) { int i = 0; int k = 0; int ccount = 0; int pcount = 0; double[,] trnxy = new double[0,0]; double[,] valxy = new double[0,0]; int trnsize = 0; int valsize = 0; int tmpinfo = 0; mlpreport tmprep = new mlpreport(); mlpbase.modelerrors moderr = new mlpbase.modelerrors(); int nin = 0; int nout = 0; int wcount = 0; int i_ = 0; int i1_ = 0; info = 0; nin = mlpbase.mlpgetinputscount(ensemble.network, _params); nout = mlpbase.mlpgetoutputscount(ensemble.network, _params); wcount = mlpbase.mlpgetweightscount(ensemble.network, _params); if( (npoints<2 || restarts<1) || (double)(decay)<(double)(0) ) { info = -1; return; } if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { for(i=0; i<=npoints-1; i++) { if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout ) { info = -2; return; } } } info = 6; // // allocate // if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { ccount = nin+1; pcount = nin; } else { ccount = nin+nout; pcount = nin+nout; } trnxy = new double[npoints, ccount]; valxy = new double[npoints, ccount]; rep.ngrad = 0; rep.nhess = 0; rep.ncholesky = 0; // // train networks // for(k=0; k<=ensemble.ensemblesize-1; k++) { // // Split set // do { trnsize = 0; valsize = 0; for(i=0; i<=npoints-1; i++) { if( (double)(math.randomreal())<(double)(0.66) ) { // // Assign sample to training set // for(i_=0; i_<=ccount-1;i_++) { trnxy[trnsize,i_] = xy[i,i_]; } trnsize = trnsize+1; } else { // // Assign sample to validation set // for(i_=0; i_<=ccount-1;i_++) { valxy[valsize,i_] = xy[i,i_]; } valsize = valsize+1; } } } while( !(trnsize!=0 && valsize!=0) ); // // Train // mlptraines(ensemble.network, trnxy, trnsize, valxy, valsize, decay, restarts, ref tmpinfo, tmprep, _params); if( tmpinfo<0 ) { info = tmpinfo; return; } // // save results // i1_ = (0) - (k*wcount); for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++) { ensemble.weights[i_] = ensemble.network.weights[i_+i1_]; } i1_ = (0) - (k*pcount); for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++) { ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_]; } i1_ = (0) - (k*pcount); for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++) { ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_]; } rep.ngrad = rep.ngrad+tmprep.ngrad; rep.nhess = rep.nhess+tmprep.nhess; rep.ncholesky = rep.ncholesky+tmprep.ncholesky; } mlpe.mlpeallerrorsx(ensemble, xy, ensemble.network.dummysxy, npoints, 0, ensemble.network.dummyidx, 0, npoints, 0, ensemble.network.buf, moderr, _params); rep.relclserror = moderr.relclserror; rep.avgce = moderr.avgce; rep.rmserror = moderr.rmserror; rep.avgerror = moderr.avgerror; rep.avgrelerror = moderr.avgrelerror; } /************************************************************************* This function trains neural network ensemble passed to this function using current dataset and early stopping training algorithm. Each early stopping round performs NRestarts random restarts (thus, EnsembleSize*NRestarts training rounds is performed in total). ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - trainer object; Ensemble - neural network ensemble. It must have same number of inputs and outputs/classes as was specified during creation of the trainer object. NRestarts - number of restarts, >=0: * NRestarts>0 means that specified number of random restarts are performed during each ES round; * NRestarts=0 is silently replaced by 1. OUTPUT PARAMETERS: Ensemble - trained ensemble; Rep - it contains all type of errors. NOTE: this training method uses BOTH early stopping and weight decay! So, you should select weight decay before starting training just as you select it before training "conventional" networks. NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(), or single-point dataset was passed, ensemble is filled by zero values. NOTE: this method uses sum-of-squares error function for training. -- ALGLIB -- Copyright 22.08.2012 by Bochkanov Sergey *************************************************************************/ public static void mlptrainensemblees(mlptrainer s, mlpe.mlpensemble ensemble, int nrestarts, mlpreport rep, alglib.xparams _params) { int nin = 0; int nout = 0; int ntype = 0; int ttype = 0; alglib.smp.shared_pool esessions = new alglib.smp.shared_pool(); apserv.sinteger sgrad = new apserv.sinteger(); mlpbase.modelerrors tmprep = new mlpbase.modelerrors(); alglib.ap.assert(s.npoints>=0, "MLPTrainEnsembleES: parameter S is not initialized or is spoiled(S.NPoints<0)"); if( !mlpe.mlpeissoftmax(ensemble, _params) ) { ntype = 0; } else { ntype = 1; } if( s.rcpar ) { ttype = 0; } else { ttype = 1; } alglib.ap.assert(ntype==ttype, "MLPTrainEnsembleES: internal error - type of input network is not similar to network type in trainer object"); nin = mlpbase.mlpgetinputscount(ensemble.network, _params); alglib.ap.assert(s.nin==nin, "MLPTrainEnsembleES: number of inputs in trainer is not equal to number of inputs in ensemble network"); nout = mlpbase.mlpgetoutputscount(ensemble.network, _params); alglib.ap.assert(s.nout==nout, "MLPTrainEnsembleES: number of outputs in trainer is not equal to number of outputs in ensemble network"); alglib.ap.assert(nrestarts>=0, "MLPTrainEnsembleES: NRestarts<0."); // // Initialize parameter Rep // rep.relclserror = 0; rep.avgce = 0; rep.rmserror = 0; rep.avgerror = 0; rep.avgrelerror = 0; rep.ngrad = 0; rep.nhess = 0; rep.ncholesky = 0; // // Allocate // apserv.ivectorsetlengthatleast(ref s.subset, s.npoints, _params); apserv.ivectorsetlengthatleast(ref s.valsubset, s.npoints, _params); // // Start training // // NOTE: ESessions is not initialized because MLPTrainEnsembleX // needs uninitialized pool. // sgrad.val = 0; mlptrainensemblex(s, ensemble, 0, ensemble.ensemblesize, nrestarts, 0, sgrad, true, esessions, _params); rep.ngrad = sgrad.val; // // Calculate errors. // if( s.datatype==0 ) { mlpe.mlpeallerrorsx(ensemble, s.densexy, s.sparsexy, s.npoints, 0, ensemble.network.dummyidx, 0, s.npoints, 0, ensemble.network.buf, tmprep, _params); } if( s.datatype==1 ) { mlpe.mlpeallerrorsx(ensemble, s.densexy, s.sparsexy, s.npoints, 1, ensemble.network.dummyidx, 0, s.npoints, 0, ensemble.network.buf, tmprep, _params); } rep.relclserror = tmprep.relclserror; rep.avgce = tmprep.avgce; rep.rmserror = tmprep.rmserror; rep.avgerror = tmprep.avgerror; rep.avgrelerror = tmprep.avgrelerror; } /************************************************************************* Internal cross-validation subroutine *************************************************************************/ private static void mlpkfoldcvgeneral(mlpbase.multilayerperceptron n, double[,] xy, int npoints, double decay, int restarts, int foldscount, bool lmalgorithm, double wstep, int maxits, ref int info, mlpreport rep, mlpcvreport cvrep, alglib.xparams _params) { int i = 0; int fold = 0; int j = 0; int k = 0; mlpbase.multilayerperceptron network = new mlpbase.multilayerperceptron(); int nin = 0; int nout = 0; int rowlen = 0; int wcount = 0; int nclasses = 0; int tssize = 0; int cvssize = 0; double[,] cvset = new double[0,0]; double[,] testset = new double[0,0]; int[] folds = new int[0]; int relcnt = 0; mlpreport internalrep = new mlpreport(); double[] x = new double[0]; double[] y = new double[0]; int i_ = 0; info = 0; // // Read network geometry, test parameters // mlpbase.mlpproperties(n, ref nin, ref nout, ref wcount, _params); if( mlpbase.mlpissoftmax(n, _params) ) { nclasses = nout; rowlen = nin+1; } else { nclasses = -nout; rowlen = nin+nout; } if( (npoints<=0 || foldscount<2) || foldscount>npoints ) { info = -1; return; } mlpbase.mlpcopy(n, network, _params); // // K-fold out cross-validation. // First, estimate generalization error // testset = new double[npoints-1+1, rowlen-1+1]; cvset = new double[npoints-1+1, rowlen-1+1]; x = new double[nin-1+1]; y = new double[nout-1+1]; mlpkfoldsplit(xy, npoints, nclasses, foldscount, false, ref folds, _params); cvrep.relclserror = 0; cvrep.avgce = 0; cvrep.rmserror = 0; cvrep.avgerror = 0; cvrep.avgrelerror = 0; rep.ngrad = 0; rep.nhess = 0; rep.ncholesky = 0; relcnt = 0; for(fold=0; fold<=foldscount-1; fold++) { // // Separate set // tssize = 0; cvssize = 0; for(i=0; i<=npoints-1; i++) { if( folds[i]==fold ) { for(i_=0; i_<=rowlen-1;i_++) { testset[tssize,i_] = xy[i,i_]; } tssize = tssize+1; } else { for(i_=0; i_<=rowlen-1;i_++) { cvset[cvssize,i_] = xy[i,i_]; } cvssize = cvssize+1; } } // // Train on CV training set // if( lmalgorithm ) { mlptrainlm(network, cvset, cvssize, decay, restarts, ref info, internalrep, _params); } else { mlptrainlbfgs(network, cvset, cvssize, decay, restarts, wstep, maxits, ref info, internalrep, _params); } if( info<0 ) { cvrep.relclserror = 0; cvrep.avgce = 0; cvrep.rmserror = 0; cvrep.avgerror = 0; cvrep.avgrelerror = 0; return; } rep.ngrad = rep.ngrad+internalrep.ngrad; rep.nhess = rep.nhess+internalrep.nhess; rep.ncholesky = rep.ncholesky+internalrep.ncholesky; // // Estimate error using CV test set // if( mlpbase.mlpissoftmax(network, _params) ) { // // classification-only code // cvrep.relclserror = cvrep.relclserror+mlpbase.mlpclserror(network, testset, tssize, _params); cvrep.avgce = cvrep.avgce+mlpbase.mlperrorn(network, testset, tssize, _params); } for(i=0; i<=tssize-1; i++) { for(i_=0; i_<=nin-1;i_++) { x[i_] = testset[i,i_]; } mlpbase.mlpprocess(network, x, ref y, _params); if( mlpbase.mlpissoftmax(network, _params) ) { // // Classification-specific code // k = (int)Math.Round(testset[i,nin]); for(j=0; j<=nout-1; j++) { if( j==k ) { cvrep.rmserror = cvrep.rmserror+math.sqr(y[j]-1); cvrep.avgerror = cvrep.avgerror+Math.Abs(y[j]-1); cvrep.avgrelerror = cvrep.avgrelerror+Math.Abs(y[j]-1); relcnt = relcnt+1; } else { cvrep.rmserror = cvrep.rmserror+math.sqr(y[j]); cvrep.avgerror = cvrep.avgerror+Math.Abs(y[j]); } } } else { // // Regression-specific code // for(j=0; j<=nout-1; j++) { cvrep.rmserror = cvrep.rmserror+math.sqr(y[j]-testset[i,nin+j]); cvrep.avgerror = cvrep.avgerror+Math.Abs(y[j]-testset[i,nin+j]); if( (double)(testset[i,nin+j])!=(double)(0) ) { cvrep.avgrelerror = cvrep.avgrelerror+Math.Abs((y[j]-testset[i,nin+j])/testset[i,nin+j]); relcnt = relcnt+1; } } } } } if( mlpbase.mlpissoftmax(network, _params) ) { cvrep.relclserror = cvrep.relclserror/npoints; cvrep.avgce = cvrep.avgce/(Math.Log(2)*npoints); } cvrep.rmserror = Math.Sqrt(cvrep.rmserror/(npoints*nout)); cvrep.avgerror = cvrep.avgerror/(npoints*nout); if( relcnt>0 ) { cvrep.avgrelerror = cvrep.avgrelerror/relcnt; } info = 1; } /************************************************************************* Subroutine prepares K-fold split of the training set. NOTES: "NClasses>0" means that we have classification task. "NClasses<0" means regression task with -NClasses real outputs. *************************************************************************/ private static void mlpkfoldsplit(double[,] xy, int npoints, int nclasses, int foldscount, bool stratifiedsplits, ref int[] folds, alglib.xparams _params) { int i = 0; int j = 0; int k = 0; hqrnd.hqrndstate rs = new hqrnd.hqrndstate(); folds = new int[0]; // // test parameters // alglib.ap.assert(npoints>0, "MLPKFoldSplit: wrong NPoints!"); alglib.ap.assert(nclasses>1 || nclasses<0, "MLPKFoldSplit: wrong NClasses!"); alglib.ap.assert(foldscount>=2 && foldscount<=npoints, "MLPKFoldSplit: wrong FoldsCount!"); alglib.ap.assert(!stratifiedsplits, "MLPKFoldSplit: stratified splits are not supported!"); // // Folds // hqrnd.hqrndrandomize(rs, _params); folds = new int[npoints-1+1]; for(i=0; i<=npoints-1; i++) { folds[i] = i*foldscount/npoints; } for(i=0; i<=npoints-2; i++) { j = i+hqrnd.hqrnduniformi(rs, npoints-i, _params); if( j!=i ) { k = folds[i]; folds[i] = folds[j]; folds[j] = k; } } } /************************************************************************* Internal subroutine for parallelization function MLPFoldCV. INPUT PARAMETERS: S - trainer object; RowSize - row size(eitherNIn+NOut or NIn+1); NRestarts - number of restarts(>=0); Folds - cross-validation set; Fold - the number of first cross-validation(>=0); DFold - the number of second cross-validation(>=Fold+1); CVY - parameter which stores the result is returned by network, training on I-th cross-validation set. It has to be preallocated. PoolDataCV- parameter for parallelization. WCount - number of weights in network, used to make decisions on parallelization. NOTE: There are no checks on the parameters correctness. -- ALGLIB -- Copyright 25.09.2012 by Bochkanov Sergey *************************************************************************/ private static void mthreadcv(mlptrainer s, int rowsize, int nrestarts, int[] folds, int fold, int dfold, double[,] cvy, alglib.smp.shared_pool pooldatacv, int wcount, alglib.xparams _params) { mlpparallelizationcv datacv = null; int i = 0; int i_ = 0; if( fold==dfold-1 ) { // // Separate set // alglib.smp.ae_shared_pool_retrieve(pooldatacv, ref datacv); datacv.subsetsize = 0; for(i=0; i<=s.npoints-1; i++) { if( folds[i]!=fold ) { datacv.subset[datacv.subsetsize] = i; datacv.subsetsize = datacv.subsetsize+1; } } // // Train on CV training set // mlptrainnetworkx(s, nrestarts, -1, datacv.subset, datacv.subsetsize, datacv.subset, 0, datacv.network, datacv.rep, true, datacv.trnpool, _params); datacv.ngrad = datacv.ngrad+datacv.rep.ngrad; // // Estimate error using CV test set // for(i=0; i<=s.npoints-1; i++) { if( folds[i]==fold ) { if( s.datatype==0 ) { for(i_=0; i_<=rowsize-1;i_++) { datacv.xyrow[i_] = s.densexy[i,i_]; } } if( s.datatype==1 ) { sparse.sparsegetrow(s.sparsexy, i, ref datacv.xyrow, _params); } mlpbase.mlpprocess(datacv.network, datacv.xyrow, ref datacv.y, _params); for(i_=0; i_<=s.nout-1;i_++) { cvy[i,i_] = datacv.y[i_]; } } } alglib.smp.ae_shared_pool_recycle(pooldatacv, ref datacv); } else { alglib.ap.assert(foldDFold-1)."); // // We expect that minimum number of iterations before convergence is 100. // Hence is our approach to evaluation of task complexity. // if( (double)(Math.Max(nrestarts, 1)*apserv.rmul3(2*wcount, s.npoints, 100, _params))>=(double)(apserv.smpactivationlevel(_params)) ) { if( _trypexec_mthreadcv(s,rowsize,nrestarts,folds,fold,dfold,cvy,pooldatacv,wcount, _params) ) { return; } } // // Split task // mthreadcv(s, rowsize, nrestarts, folds, fold, (fold+dfold)/2, cvy, pooldatacv, wcount, _params); mthreadcv(s, rowsize, nrestarts, folds, (fold+dfold)/2, dfold, cvy, pooldatacv, wcount, _params); } } /************************************************************************* Serial stub for GPL edition. *************************************************************************/ public static bool _trypexec_mthreadcv(mlptrainer s, int rowsize, int nrestarts, int[] folds, int fold, int dfold, double[,] cvy, alglib.smp.shared_pool pooldatacv, int wcount, alglib.xparams _params) { return false; } /************************************************************************* This function trains neural network passed to this function, using current dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset()) and current training settings. Training from NRestarts random starting positions is performed, best network is chosen. This function is inteded to be used internally. It may be used in several settings: * training with ValSubsetSize=0, corresponds to "normal" training with termination criteria based on S.MaxIts (steps count) and S.WStep (step size). Training sample is given by TrnSubset/TrnSubsetSize. * training with ValSubsetSize>0, corresponds to early stopping training with additional MaxIts/WStep stopping criteria. Training sample is given by TrnSubset/TrnSubsetSize, validation sample is given by ValSubset/ ValSubsetSize. -- ALGLIB -- Copyright 13.08.2012 by Bochkanov Sergey *************************************************************************/ private static void mlptrainnetworkx(mlptrainer s, int nrestarts, int algokind, int[] trnsubset, int trnsubsetsize, int[] valsubset, int valsubsetsize, mlpbase.multilayerperceptron network, mlpreport rep, bool isrootcall, alglib.smp.shared_pool sessions, alglib.xparams _params) { mlpbase.modelerrors modrep = new mlpbase.modelerrors(); double eval = 0; double ebest = 0; int ngradbatch = 0; int nin = 0; int nout = 0; int wcount = 0; int pcount = 0; int itbest = 0; int itcnt = 0; int ntype = 0; int ttype = 0; bool rndstart = new bool(); int i = 0; int nr0 = 0; int nr1 = 0; mlpreport rep0 = new mlpreport(); mlpreport rep1 = new mlpreport(); bool randomizenetwork = new bool(); double bestrmserror = 0; smlptrnsession psession = null; int i_ = 0; mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount, _params); // // Process root call // if( isrootcall ) { // // Try parallelization // We expect that minimum number of iterations before convergence is 100. // Hence is our approach to evaluation of task complexity. // if( (double)(Math.Max(nrestarts, 1)*apserv.rmul3(2*wcount, s.npoints, 100, _params))>=(double)(apserv.smpactivationlevel(_params)) ) { if( _trypexec_mlptrainnetworkx(s,nrestarts,algokind,trnsubset,trnsubsetsize,valsubset,valsubsetsize,network,rep,isrootcall,sessions, _params) ) { return; } } // // Check correctness of parameters // alglib.ap.assert(algokind==0 || algokind==-1, "MLPTrainNetworkX: unexpected AlgoKind"); alglib.ap.assert(s.npoints>=0, "MLPTrainNetworkX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)"); if( s.rcpar ) { ttype = 0; } else { ttype = 1; } if( !mlpbase.mlpissoftmax(network, _params) ) { ntype = 0; } else { ntype = 1; } alglib.ap.assert(ntype==ttype, "MLPTrainNetworkX: internal error - type of the training network is not similar to network type in trainer object"); alglib.ap.assert(s.nin==nin, "MLPTrainNetworkX: internal error - number of inputs in trainer is not equal to number of inputs in the training network."); alglib.ap.assert(s.nout==nout, "MLPTrainNetworkX: internal error - number of outputs in trainer is not equal to number of outputs in the training network."); alglib.ap.assert(nrestarts>=0, "MLPTrainNetworkX: internal error - NRestarts<0."); alglib.ap.assert(alglib.ap.len(trnsubset)>=trnsubsetsize, "MLPTrainNetworkX: internal error - parameter TrnSubsetSize more than input subset size(Length(TrnSubset)=0 && trnsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter TrnSubset contains incorrect index(TrnSubset[I]<0 or TrnSubset[I]>S.NPoints-1)"); } alglib.ap.assert(alglib.ap.len(valsubset)>=valsubsetsize, "MLPTrainNetworkX: internal error - parameter ValSubsetSize more than input subset size(Length(ValSubset)=0 && valsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter ValSubset contains incorrect index(ValSubset[I]<0 or ValSubset[I]>S.NPoints-1)"); } // // Train // randomizenetwork = nrestarts>0; initmlptrnsessions(network, randomizenetwork, s, sessions, _params); mlptrainnetworkx(s, nrestarts, algokind, trnsubset, trnsubsetsize, valsubset, valsubsetsize, network, rep, false, sessions, _params); // // Choose best network // bestrmserror = math.maxrealnumber; alglib.smp.ae_shared_pool_first_recycled(sessions, ref psession); while( psession!=null ) { if( (double)(psession.bestrmserror)<(double)(bestrmserror) ) { mlpbase.mlpimporttunableparameters(network, psession.bestparameters, _params); bestrmserror = psession.bestrmserror; } alglib.smp.ae_shared_pool_next_recycled(sessions, ref psession); } // // Calculate errors // if( s.datatype==0 ) { mlpbase.mlpallerrorssubset(network, s.densexy, s.npoints, trnsubset, trnsubsetsize, modrep, _params); } if( s.datatype==1 ) { mlpbase.mlpallerrorssparsesubset(network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize, modrep, _params); } rep.relclserror = modrep.relclserror; rep.avgce = modrep.avgce; rep.rmserror = modrep.rmserror; rep.avgerror = modrep.avgerror; rep.avgrelerror = modrep.avgrelerror; // // Done // return; } // // Split problem, if we have more than 1 restart // if( nrestarts>=2 ) { // // Divide problem with NRestarts into two: NR0 and NR1. // nr0 = nrestarts/2; nr1 = nrestarts-nr0; mlptrainnetworkx(s, nr0, algokind, trnsubset, trnsubsetsize, valsubset, valsubsetsize, network, rep0, false, sessions, _params); mlptrainnetworkx(s, nr1, algokind, trnsubset, trnsubsetsize, valsubset, valsubsetsize, network, rep1, false, sessions, _params); // // Aggregate results // rep.ngrad = rep0.ngrad+rep1.ngrad; rep.nhess = rep0.nhess+rep1.nhess; rep.ncholesky = rep0.ncholesky+rep1.ncholesky; // // Done :) // return; } // // Execution with NRestarts=1 or NRestarts=0: // * NRestarts=1 means that network is restarted from random position // * NRestarts=0 means that network is not randomized // alglib.ap.assert(nrestarts==0 || nrestarts==1, "MLPTrainNetworkX: internal error"); rep.ngrad = 0; rep.nhess = 0; rep.ncholesky = 0; alglib.smp.ae_shared_pool_retrieve(sessions, ref psession); if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && trnsubsetsize!=0 ) { // // Train network using combination of early stopping and step-size // and step-count based criteria. Network state with best value of // validation set error is stored in WBuf0. When validation set is // zero, most recent state of network is stored. // rndstart = nrestarts!=0; ngradbatch = 0; eval = 0; ebest = 0; itbest = 0; itcnt = 0; mlpstarttrainingx(s, rndstart, algokind, trnsubset, trnsubsetsize, psession, _params); if( s.datatype==0 ) { ebest = mlpbase.mlperrorsubset(psession.network, s.densexy, s.npoints, valsubset, valsubsetsize, _params); } if( s.datatype==1 ) { ebest = mlpbase.mlperrorsparsesubset(psession.network, s.sparsexy, s.npoints, valsubset, valsubsetsize, _params); } for(i_=0; i_<=wcount-1;i_++) { psession.wbuf0[i_] = psession.network.weights[i_]; } while( mlpcontinuetrainingx(s, trnsubset, trnsubsetsize, ref ngradbatch, psession, _params) ) { if( s.datatype==0 ) { eval = mlpbase.mlperrorsubset(psession.network, s.densexy, s.npoints, valsubset, valsubsetsize, _params); } if( s.datatype==1 ) { eval = mlpbase.mlperrorsparsesubset(psession.network, s.sparsexy, s.npoints, valsubset, valsubsetsize, _params); } if( (double)(eval)<=(double)(ebest) || valsubsetsize==0 ) { for(i_=0; i_<=wcount-1;i_++) { psession.wbuf0[i_] = psession.network.weights[i_]; } ebest = eval; itbest = itcnt; } if( itcnt>30 && (double)(itcnt)>(double)(1.5*itbest) ) { break; } itcnt = itcnt+1; } for(i_=0; i_<=wcount-1;i_++) { psession.network.weights[i_] = psession.wbuf0[i_]; } rep.ngrad = ngradbatch; } else { for(i=0; i<=wcount-1; i++) { psession.network.weights[i] = 0; } } // // Evaluate network performance and update PSession.BestParameters/BestRMSError // (if needed). // if( s.datatype==0 ) { mlpbase.mlpallerrorssubset(psession.network, s.densexy, s.npoints, trnsubset, trnsubsetsize, modrep, _params); } if( s.datatype==1 ) { mlpbase.mlpallerrorssparsesubset(psession.network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize, modrep, _params); } if( (double)(modrep.rmserror)<(double)(psession.bestrmserror) ) { mlpbase.mlpexporttunableparameters(psession.network, ref psession.bestparameters, ref pcount, _params); psession.bestrmserror = modrep.rmserror; } // // Move session back to pool // alglib.smp.ae_shared_pool_recycle(sessions, ref psession); } /************************************************************************* Serial stub for GPL edition. *************************************************************************/ public static bool _trypexec_mlptrainnetworkx(mlptrainer s, int nrestarts, int algokind, int[] trnsubset, int trnsubsetsize, int[] valsubset, int valsubsetsize, mlpbase.multilayerperceptron network, mlpreport rep, bool isrootcall, alglib.smp.shared_pool sessions, alglib.xparams _params) { return false; } /************************************************************************* This function trains neural network ensemble passed to this function using current dataset and early stopping training algorithm. Each early stopping round performs NRestarts random restarts (thus, EnsembleSize*NRestarts training rounds is performed in total). -- ALGLIB -- Copyright 22.08.2012 by Bochkanov Sergey *************************************************************************/ private static void mlptrainensemblex(mlptrainer s, mlpe.mlpensemble ensemble, int idx0, int idx1, int nrestarts, int trainingmethod, apserv.sinteger ngrad, bool isrootcall, alglib.smp.shared_pool esessions, alglib.xparams _params) { int pcount = 0; int nin = 0; int nout = 0; int wcount = 0; int i = 0; int j = 0; int k = 0; int trnsubsetsize = 0; int valsubsetsize = 0; int k0 = 0; apserv.sinteger ngrad0 = new apserv.sinteger(); apserv.sinteger ngrad1 = new apserv.sinteger(); mlpetrnsession psession = null; hqrnd.hqrndstate rs = new hqrnd.hqrndstate(); int i_ = 0; int i1_ = 0; nin = mlpbase.mlpgetinputscount(ensemble.network, _params); nout = mlpbase.mlpgetoutputscount(ensemble.network, _params); wcount = mlpbase.mlpgetweightscount(ensemble.network, _params); if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { pcount = nin; } else { pcount = nin+nout; } if( nrestarts<=0 ) { nrestarts = 1; } // // Handle degenerate case // if( s.npoints<2 ) { for(i=idx0; i<=idx1-1; i++) { for(j=0; j<=wcount-1; j++) { ensemble.weights[i*wcount+j] = 0.0; } for(j=0; j<=pcount-1; j++) { ensemble.columnmeans[i*pcount+j] = 0.0; ensemble.columnsigmas[i*pcount+j] = 1.0; } } return; } // // Process root call // if( isrootcall ) { // // Try parallelization // We expect that minimum number of iterations before convergence is 100. // Hence is our approach to evaluation of task complexity. // if( (double)(Math.Max(nrestarts, 1)*(idx1-idx0)*apserv.rmul3(2*wcount, s.npoints, 100, _params))>=(double)(apserv.smpactivationlevel(_params)) ) { if( _trypexec_mlptrainensemblex(s,ensemble,idx0,idx1,nrestarts,trainingmethod,ngrad,isrootcall,esessions, _params) ) { return; } } // // Prepare: // * prepare MLPETrnSessions // * fill ensemble by zeros (helps to detect errors) // initmlpetrnsessions(ensemble.network, s, esessions, _params); for(i=idx0; i<=idx1-1; i++) { for(j=0; j<=wcount-1; j++) { ensemble.weights[i*wcount+j] = 0.0; } for(j=0; j<=pcount-1; j++) { ensemble.columnmeans[i*pcount+j] = 0.0; ensemble.columnsigmas[i*pcount+j] = 0.0; } } // // Train in non-root mode and exit // mlptrainensemblex(s, ensemble, idx0, idx1, nrestarts, trainingmethod, ngrad, false, esessions, _params); return; } // // Split problem // if( idx1-idx0>=2 ) { k0 = (idx1-idx0)/2; ngrad0.val = 0; ngrad1.val = 0; mlptrainensemblex(s, ensemble, idx0, idx0+k0, nrestarts, trainingmethod, ngrad0, false, esessions, _params); mlptrainensemblex(s, ensemble, idx0+k0, idx1, nrestarts, trainingmethod, ngrad1, false, esessions, _params); ngrad.val = ngrad0.val+ngrad1.val; return; } // // Retrieve and prepare session // alglib.smp.ae_shared_pool_retrieve(esessions, ref psession); // // Train // hqrnd.hqrndrandomize(rs, _params); for(k=idx0; k<=idx1-1; k++) { // // Split set // trnsubsetsize = 0; valsubsetsize = 0; if( trainingmethod==0 ) { do { trnsubsetsize = 0; valsubsetsize = 0; for(i=0; i<=s.npoints-1; i++) { if( (double)(math.randomreal())<(double)(0.66) ) { // // Assign sample to training set // psession.trnsubset[trnsubsetsize] = i; trnsubsetsize = trnsubsetsize+1; } else { // // Assign sample to validation set // psession.valsubset[valsubsetsize] = i; valsubsetsize = valsubsetsize+1; } } } while( !(trnsubsetsize!=0 && valsubsetsize!=0) ); } if( trainingmethod==1 ) { valsubsetsize = 0; trnsubsetsize = s.npoints; for(i=0; i<=s.npoints-1; i++) { psession.trnsubset[i] = hqrnd.hqrnduniformi(rs, s.npoints, _params); } } // // Train // mlptrainnetworkx(s, nrestarts, -1, psession.trnsubset, trnsubsetsize, psession.valsubset, valsubsetsize, psession.network, psession.mlprep, true, psession.mlpsessions, _params); ngrad.val = ngrad.val+psession.mlprep.ngrad; // // Save results // i1_ = (0) - (k*wcount); for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++) { ensemble.weights[i_] = psession.network.weights[i_+i1_]; } i1_ = (0) - (k*pcount); for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++) { ensemble.columnmeans[i_] = psession.network.columnmeans[i_+i1_]; } i1_ = (0) - (k*pcount); for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++) { ensemble.columnsigmas[i_] = psession.network.columnsigmas[i_+i1_]; } } // // Recycle session // alglib.smp.ae_shared_pool_recycle(esessions, ref psession); } /************************************************************************* Serial stub for GPL edition. *************************************************************************/ public static bool _trypexec_mlptrainensemblex(mlptrainer s, mlpe.mlpensemble ensemble, int idx0, int idx1, int nrestarts, int trainingmethod, apserv.sinteger ngrad, bool isrootcall, alglib.smp.shared_pool esessions, alglib.xparams _params) { return false; } /************************************************************************* This function performs step-by-step training of the neural network. Here "step-by-step" means that training starts with MLPStartTrainingX call, and then user subsequently calls MLPContinueTrainingX to perform one more iteration of the training. After call to this function trainer object remembers network and is ready to train it. However, no training is performed until first call to MLPContinueTraining() function. Subsequent calls to MLPContinueTraining() will advance traing progress one iteration further. -- ALGLIB -- Copyright 13.08.2012 by Bochkanov Sergey *************************************************************************/ private static void mlpstarttrainingx(mlptrainer s, bool randomstart, int algokind, int[] subset, int subsetsize, smlptrnsession session, alglib.xparams _params) { int nin = 0; int nout = 0; int wcount = 0; int ntype = 0; int ttype = 0; int i = 0; // // Check parameters // alglib.ap.assert(s.npoints>=0, "MLPStartTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)"); alglib.ap.assert(algokind==0 || algokind==-1, "MLPStartTrainingX: unexpected AlgoKind"); if( s.rcpar ) { ttype = 0; } else { ttype = 1; } if( !mlpbase.mlpissoftmax(session.network, _params) ) { ntype = 0; } else { ntype = 1; } alglib.ap.assert(ntype==ttype, "MLPStartTrainingX: internal error - type of the resulting network is not similar to network type in trainer object"); mlpbase.mlpproperties(session.network, ref nin, ref nout, ref wcount, _params); alglib.ap.assert(s.nin==nin, "MLPStartTrainingX: number of inputs in trainer is not equal to number of inputs in the network."); alglib.ap.assert(s.nout==nout, "MLPStartTrainingX: number of outputs in trainer is not equal to number of outputs in the network."); alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPStartTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)=0 && subset[i]<=s.npoints-1, "MLPStartTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1)"); } // // Prepare session // minlbfgs.minlbfgssetcond(session.optimizer, 0.0, 0.0, s.wstep, s.maxits, _params); if( s.npoints>0 && subsetsize!=0 ) { if( randomstart ) { mlpbase.mlprandomize(session.network, _params); } minlbfgs.minlbfgsrestartfrom(session.optimizer, session.network.weights, _params); } else { for(i=0; i<=wcount-1; i++) { session.network.weights[i] = 0; } } if( algokind==-1 ) { session.algoused = s.algokind; if( s.algokind==1 ) { session.minibatchsize = s.minibatchsize; } } else { session.algoused = 0; } hqrnd.hqrndrandomize(session.generator, _params); session.rstate.ia = new int[15+1]; session.rstate.ra = new double[1+1]; session.rstate.stage = -1; } /************************************************************************* This function performs step-by-step training of the neural network. Here "step-by-step" means that training starts with MLPStartTrainingX call, and then user subsequently calls MLPContinueTrainingX to perform one more iteration of the training. This function performs one more iteration of the training and returns either True (training continues) or False (training stopped). In case True was returned, Network weights are updated according to the current state of the optimization progress. In case False was returned, no additional updates is performed (previous update of the network weights moved us to the final point, and no additional updates is needed). EXAMPLE: > > [initialize network and trainer object] > > MLPStartTraining(Trainer, Network, True) > while MLPContinueTraining(Trainer, Network) do > [visualize training progress] > -- ALGLIB -- Copyright 13.08.2012 by Bochkanov Sergey *************************************************************************/ private static bool mlpcontinuetrainingx(mlptrainer s, int[] subset, int subsetsize, ref int ngradbatch, smlptrnsession session, alglib.xparams _params) { bool result = new bool(); int nin = 0; int nout = 0; int wcount = 0; int twcount = 0; int ntype = 0; int ttype = 0; double decay = 0; double v = 0; int i = 0; int j = 0; int k = 0; int trnsetsize = 0; int epoch = 0; int minibatchcount = 0; int minibatchidx = 0; int cursize = 0; int idx0 = 0; int idx1 = 0; int i_ = 0; // // Reverse communication preparations // I know it looks ugly, but it works the same way // anywhere from C++ to Python. // // This code initializes locals by: // * random values determined during code // generation - on first subroutine call // * values from previous call - on subsequent calls // if( session.rstate.stage>=0 ) { nin = session.rstate.ia[0]; nout = session.rstate.ia[1]; wcount = session.rstate.ia[2]; twcount = session.rstate.ia[3]; ntype = session.rstate.ia[4]; ttype = session.rstate.ia[5]; i = session.rstate.ia[6]; j = session.rstate.ia[7]; k = session.rstate.ia[8]; trnsetsize = session.rstate.ia[9]; epoch = session.rstate.ia[10]; minibatchcount = session.rstate.ia[11]; minibatchidx = session.rstate.ia[12]; cursize = session.rstate.ia[13]; idx0 = session.rstate.ia[14]; idx1 = session.rstate.ia[15]; decay = session.rstate.ra[0]; v = session.rstate.ra[1]; } else { nin = 359; nout = -58; wcount = -919; twcount = -909; ntype = 81; ttype = 255; i = 74; j = -788; k = 809; trnsetsize = 205; epoch = -838; minibatchcount = 939; minibatchidx = -526; cursize = 763; idx0 = -541; idx1 = -698; decay = -900; v = -318; } if( session.rstate.stage==0 ) { goto lbl_0; } // // Routine body // // // Check correctness of inputs // alglib.ap.assert(s.npoints>=0, "MLPContinueTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)."); if( s.rcpar ) { ttype = 0; } else { ttype = 1; } if( !mlpbase.mlpissoftmax(session.network, _params) ) { ntype = 0; } else { ntype = 1; } alglib.ap.assert(ntype==ttype, "MLPContinueTrainingX: internal error - type of the resulting network is not similar to network type in trainer object."); mlpbase.mlpproperties(session.network, ref nin, ref nout, ref wcount, _params); alglib.ap.assert(s.nin==nin, "MLPContinueTrainingX: internal error - number of inputs in trainer is not equal to number of inputs in the network."); alglib.ap.assert(s.nout==nout, "MLPContinueTrainingX: internal error - number of outputs in trainer is not equal to number of outputs in the network."); alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPContinueTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)=0 && subset[i]<=s.npoints-1, "MLPContinueTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1)."); } // // Quick exit on empty training set // if( s.npoints==0 || subsetsize==0 ) { result = false; return result; } // // Minibatch training // if( session.algoused==1 ) { alglib.ap.assert(false, "MINIBATCH TRAINING IS NOT IMPLEMENTED YET"); } // // Last option: full batch training // decay = s.decay; lbl_1: if( !minlbfgs.minlbfgsiteration(session.optimizer, _params) ) { goto lbl_2; } if( !session.optimizer.xupdated ) { goto lbl_3; } for(i_=0; i_<=wcount-1;i_++) { session.network.weights[i_] = session.optimizer.x[i_]; } session.rstate.stage = 0; goto lbl_rcomm; lbl_0: lbl_3: for(i_=0; i_<=wcount-1;i_++) { session.network.weights[i_] = session.optimizer.x[i_]; } if( s.datatype==0 ) { mlpbase.mlpgradbatchsubset(session.network, s.densexy, s.npoints, subset, subsetsize, ref session.optimizer.f, ref session.optimizer.g, _params); } if( s.datatype==1 ) { mlpbase.mlpgradbatchsparsesubset(session.network, s.sparsexy, s.npoints, subset, subsetsize, ref session.optimizer.f, ref session.optimizer.g, _params); } // // Increment number of operations performed on batch gradient // ngradbatch = ngradbatch+1; v = 0.0; for(i_=0; i_<=wcount-1;i_++) { v += session.network.weights[i_]*session.network.weights[i_]; } session.optimizer.f = session.optimizer.f+0.5*decay*v; for(i_=0; i_<=wcount-1;i_++) { session.optimizer.g[i_] = session.optimizer.g[i_] + decay*session.network.weights[i_]; } goto lbl_1; lbl_2: minlbfgs.minlbfgsresultsbuf(session.optimizer, ref session.network.weights, session.optimizerrep, _params); result = false; return result; // // Saving state // lbl_rcomm: result = true; session.rstate.ia[0] = nin; session.rstate.ia[1] = nout; session.rstate.ia[2] = wcount; session.rstate.ia[3] = twcount; session.rstate.ia[4] = ntype; session.rstate.ia[5] = ttype; session.rstate.ia[6] = i; session.rstate.ia[7] = j; session.rstate.ia[8] = k; session.rstate.ia[9] = trnsetsize; session.rstate.ia[10] = epoch; session.rstate.ia[11] = minibatchcount; session.rstate.ia[12] = minibatchidx; session.rstate.ia[13] = cursize; session.rstate.ia[14] = idx0; session.rstate.ia[15] = idx1; session.rstate.ra[0] = decay; session.rstate.ra[1] = v; return result; } /************************************************************************* Internal bagging subroutine. -- ALGLIB -- Copyright 19.02.2009 by Bochkanov Sergey *************************************************************************/ private static void mlpebagginginternal(mlpe.mlpensemble ensemble, double[,] xy, int npoints, double decay, int restarts, double wstep, int maxits, bool lmalgorithm, ref int info, mlpreport rep, mlpcvreport ooberrors, alglib.xparams _params) { double[,] xys = new double[0,0]; bool[] s = new bool[0]; double[,] oobbuf = new double[0,0]; int[] oobcntbuf = new int[0]; double[] x = new double[0]; double[] y = new double[0]; double[] dy = new double[0]; double[] dsbuf = new double[0]; int ccnt = 0; int pcnt = 0; int i = 0; int j = 0; int k = 0; double v = 0; mlpreport tmprep = new mlpreport(); int nin = 0; int nout = 0; int wcount = 0; hqrnd.hqrndstate rs = new hqrnd.hqrndstate(); int i_ = 0; int i1_ = 0; info = 0; nin = mlpbase.mlpgetinputscount(ensemble.network, _params); nout = mlpbase.mlpgetoutputscount(ensemble.network, _params); wcount = mlpbase.mlpgetweightscount(ensemble.network, _params); // // Test for inputs // if( (!lmalgorithm && (double)(wstep)==(double)(0)) && maxits==0 ) { info = -8; return; } if( ((npoints<=0 || restarts<1) || (double)(wstep)<(double)(0)) || maxits<0 ) { info = -1; return; } if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { for(i=0; i<=npoints-1; i++) { if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout ) { info = -2; return; } } } // // allocate temporaries // info = 2; rep.ngrad = 0; rep.nhess = 0; rep.ncholesky = 0; ooberrors.relclserror = 0; ooberrors.avgce = 0; ooberrors.rmserror = 0; ooberrors.avgerror = 0; ooberrors.avgrelerror = 0; if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { ccnt = nin+1; pcnt = nin; } else { ccnt = nin+nout; pcnt = nin+nout; } xys = new double[npoints, ccnt]; s = new bool[npoints]; oobbuf = new double[npoints, nout]; oobcntbuf = new int[npoints]; x = new double[nin]; y = new double[nout]; if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { dy = new double[1]; } else { dy = new double[nout]; } for(i=0; i<=npoints-1; i++) { for(j=0; j<=nout-1; j++) { oobbuf[i,j] = 0; } } for(i=0; i<=npoints-1; i++) { oobcntbuf[i] = 0; } // // main bagging cycle // hqrnd.hqrndrandomize(rs, _params); for(k=0; k<=ensemble.ensemblesize-1; k++) { // // prepare dataset // for(i=0; i<=npoints-1; i++) { s[i] = false; } for(i=0; i<=npoints-1; i++) { j = hqrnd.hqrnduniformi(rs, npoints, _params); s[j] = true; for(i_=0; i_<=ccnt-1;i_++) { xys[i,i_] = xy[j,i_]; } } // // train // if( lmalgorithm ) { mlptrainlm(ensemble.network, xys, npoints, decay, restarts, ref info, tmprep, _params); } else { mlptrainlbfgs(ensemble.network, xys, npoints, decay, restarts, wstep, maxits, ref info, tmprep, _params); } if( info<0 ) { return; } // // save results // rep.ngrad = rep.ngrad+tmprep.ngrad; rep.nhess = rep.nhess+tmprep.nhess; rep.ncholesky = rep.ncholesky+tmprep.ncholesky; i1_ = (0) - (k*wcount); for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++) { ensemble.weights[i_] = ensemble.network.weights[i_+i1_]; } i1_ = (0) - (k*pcnt); for(i_=k*pcnt; i_<=(k+1)*pcnt-1;i_++) { ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_]; } i1_ = (0) - (k*pcnt); for(i_=k*pcnt; i_<=(k+1)*pcnt-1;i_++) { ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_]; } // // OOB estimates // for(i=0; i<=npoints-1; i++) { if( !s[i] ) { for(i_=0; i_<=nin-1;i_++) { x[i_] = xy[i,i_]; } mlpbase.mlpprocess(ensemble.network, x, ref y, _params); for(i_=0; i_<=nout-1;i_++) { oobbuf[i,i_] = oobbuf[i,i_] + y[i_]; } oobcntbuf[i] = oobcntbuf[i]+1; } } } // // OOB estimates // if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { bdss.dserrallocate(nout, ref dsbuf, _params); } else { bdss.dserrallocate(-nout, ref dsbuf, _params); } for(i=0; i<=npoints-1; i++) { if( oobcntbuf[i]!=0 ) { v = (double)1/(double)oobcntbuf[i]; for(i_=0; i_<=nout-1;i_++) { y[i_] = v*oobbuf[i,i_]; } if( mlpbase.mlpissoftmax(ensemble.network, _params) ) { dy[0] = xy[i,nin]; } else { i1_ = (nin) - (0); for(i_=0; i_<=nout-1;i_++) { dy[i_] = v*xy[i,i_+i1_]; } } bdss.dserraccumulate(ref dsbuf, y, dy, _params); } } bdss.dserrfinish(ref dsbuf, _params); ooberrors.relclserror = dsbuf[0]; ooberrors.avgce = dsbuf[1]; ooberrors.rmserror = dsbuf[2]; ooberrors.avgerror = dsbuf[3]; ooberrors.avgrelerror = dsbuf[4]; } /************************************************************************* This function initializes temporaries needed for training session. -- ALGLIB -- Copyright 01.07.2013 by Bochkanov Sergey *************************************************************************/ private static void initmlptrnsession(mlpbase.multilayerperceptron networktrained, bool randomizenetwork, mlptrainer trainer, smlptrnsession session, alglib.xparams _params) { int nin = 0; int nout = 0; int wcount = 0; int pcount = 0; int[] dummysubset = new int[0]; // // Prepare network: // * copy input network to Session.Network // * re-initialize preprocessor and weights if RandomizeNetwork=True // mlpbase.mlpcopy(networktrained, session.network, _params); if( randomizenetwork ) { alglib.ap.assert(trainer.datatype==0 || trainer.datatype==1, "InitTemporaries: unexpected Trainer.DataType"); if( trainer.datatype==0 ) { mlpbase.mlpinitpreprocessorsubset(session.network, trainer.densexy, trainer.npoints, dummysubset, -1, _params); } if( trainer.datatype==1 ) { mlpbase.mlpinitpreprocessorsparsesubset(session.network, trainer.sparsexy, trainer.npoints, dummysubset, -1, _params); } mlpbase.mlprandomize(session.network, _params); session.randomizenetwork = true; } else { session.randomizenetwork = false; } // // Determine network geometry and initialize optimizer // mlpbase.mlpproperties(session.network, ref nin, ref nout, ref wcount, _params); minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, trainer.lbfgsfactor), session.network.weights, session.optimizer, _params); minlbfgs.minlbfgssetxrep(session.optimizer, true, _params); // // Create buffers // session.wbuf0 = new double[wcount]; session.wbuf1 = new double[wcount]; // // Initialize session result // mlpbase.mlpexporttunableparameters(session.network, ref session.bestparameters, ref pcount, _params); session.bestrmserror = math.maxrealnumber; } /************************************************************************* This function initializes temporaries needed for training session. *************************************************************************/ private static void initmlptrnsessions(mlpbase.multilayerperceptron networktrained, bool randomizenetwork, mlptrainer trainer, alglib.smp.shared_pool sessions, alglib.xparams _params) { int[] dummysubset = new int[0]; smlptrnsession t = new smlptrnsession(); smlptrnsession p = null; if( alglib.smp.ae_shared_pool_is_initialized(sessions) ) { // // Pool was already initialized. // Clear sessions stored in the pool. // alglib.smp.ae_shared_pool_first_recycled(sessions, ref p); while( p!=null ) { alglib.ap.assert(mlpbase.mlpsamearchitecture(p.network, networktrained, _params), "InitMLPTrnSessions: internal consistency error"); p.bestrmserror = math.maxrealnumber; alglib.smp.ae_shared_pool_next_recycled(sessions, ref p); } } else { // // Prepare session and seed pool // initmlptrnsession(networktrained, randomizenetwork, trainer, t, _params); alglib.smp.ae_shared_pool_set_seed(sessions, t); } } /************************************************************************* This function initializes temporaries needed for ensemble training. *************************************************************************/ private static void initmlpetrnsession(mlpbase.multilayerperceptron individualnetwork, mlptrainer trainer, mlpetrnsession session, alglib.xparams _params) { int[] dummysubset = new int[0]; // // Prepare network: // * copy input network to Session.Network // * re-initialize preprocessor and weights if RandomizeNetwork=True // mlpbase.mlpcopy(individualnetwork, session.network, _params); initmlptrnsessions(individualnetwork, true, trainer, session.mlpsessions, _params); apserv.ivectorsetlengthatleast(ref session.trnsubset, trainer.npoints, _params); apserv.ivectorsetlengthatleast(ref session.valsubset, trainer.npoints, _params); } /************************************************************************* This function initializes temporaries needed for training session. *************************************************************************/ private static void initmlpetrnsessions(mlpbase.multilayerperceptron individualnetwork, mlptrainer trainer, alglib.smp.shared_pool sessions, alglib.xparams _params) { mlpetrnsession t = new mlpetrnsession(); if( !alglib.smp.ae_shared_pool_is_initialized(sessions) ) { initmlpetrnsession(individualnetwork, trainer, t, _params); alglib.smp.ae_shared_pool_set_seed(sessions, t); } } } public class clustering { /************************************************************************* This structure is used to store temporaries for KMeansGenerateInternal function, so we will be able to reuse them during multiple subsequent calls. -- ALGLIB -- Copyright 24.07.2015 by Bochkanov Sergey *************************************************************************/ public class kmeansbuffers : apobject { public double[,] ct; public double[,] ctbest; public int[] xycbest; public int[] xycprev; public double[] d2; public int[] csizes; public apserv.apbuffers initbuf; public alglib.smp.shared_pool updatepool; public kmeansbuffers() { init(); } public override void init() { ct = new double[0,0]; ctbest = new double[0,0]; xycbest = new int[0]; xycprev = new int[0]; d2 = new double[0]; csizes = new int[0]; initbuf = new apserv.apbuffers(); updatepool = new alglib.smp.shared_pool(); } public override alglib.apobject make_copy() { kmeansbuffers _result = new kmeansbuffers(); _result.ct = (double[,])ct.Clone(); _result.ctbest = (double[,])ctbest.Clone(); _result.xycbest = (int[])xycbest.Clone(); _result.xycprev = (int[])xycprev.Clone(); _result.d2 = (double[])d2.Clone(); _result.csizes = (int[])csizes.Clone(); _result.initbuf = (apserv.apbuffers)initbuf.make_copy(); _result.updatepool = (alglib.smp.shared_pool)updatepool.make_copy(); return _result; } }; /************************************************************************* This structure is a clusterization engine. You should not try to access its fields directly. Use ALGLIB functions in order to work with this object. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public class clusterizerstate : apobject { public int npoints; public int nfeatures; public int disttype; public double[,] xy; public double[,] d; public int ahcalgo; public int kmeansrestarts; public int kmeansmaxits; public int kmeansinitalgo; public bool kmeansdbgnoits; public int seed; public double[,] tmpd; public apserv.apbuffers distbuf; public kmeansbuffers kmeanstmp; public clusterizerstate() { init(); } public override void init() { xy = new double[0,0]; d = new double[0,0]; tmpd = new double[0,0]; distbuf = new apserv.apbuffers(); kmeanstmp = new kmeansbuffers(); } public override alglib.apobject make_copy() { clusterizerstate _result = new clusterizerstate(); _result.npoints = npoints; _result.nfeatures = nfeatures; _result.disttype = disttype; _result.xy = (double[,])xy.Clone(); _result.d = (double[,])d.Clone(); _result.ahcalgo = ahcalgo; _result.kmeansrestarts = kmeansrestarts; _result.kmeansmaxits = kmeansmaxits; _result.kmeansinitalgo = kmeansinitalgo; _result.kmeansdbgnoits = kmeansdbgnoits; _result.seed = seed; _result.tmpd = (double[,])tmpd.Clone(); _result.distbuf = (apserv.apbuffers)distbuf.make_copy(); _result.kmeanstmp = (kmeansbuffers)kmeanstmp.make_copy(); return _result; } }; /************************************************************************* This structure is used to store results of the agglomerative hierarchical clustering (AHC). Following information is returned: * TerminationType - completion code: * 1 for successful completion of algorithm * -5 inappropriate combination of clustering algorithm and distance function was used. As for now, it is possible only when Ward's method is called for dataset with non-Euclidean distance function. In case negative completion code is returned, other fields of report structure are invalid and should not be used. * NPoints contains number of points in the original dataset * Z contains information about merges performed (see below). Z contains indexes from the original (unsorted) dataset and it can be used when you need to know what points were merged. However, it is not convenient when you want to build a dendrograd (see below). * if you want to build dendrogram, you can use Z, but it is not good option, because Z contains indexes from unsorted dataset. Dendrogram built from such dataset is likely to have intersections. So, you have to reorder you points before building dendrogram. Permutation which reorders point is returned in P. Another representation of merges, which is more convenient for dendorgram construction, is returned in PM. * more information on format of Z, P and PM can be found below and in the examples from ALGLIB Reference Manual. FORMAL DESCRIPTION OF FIELDS: NPoints number of points Z array[NPoints-1,2], contains indexes of clusters linked in pairs to form clustering tree. I-th row corresponds to I-th merge: * Z[I,0] - index of the first cluster to merge * Z[I,1] - index of the second cluster to merge * Z[I,0]=0 NFeatures number of variables, >=1 TerminationType completion code: * -5 if distance type is anything different from Euclidean metric * -3 for degenerate dataset: a) less than K distinct points, b) K=0 for non-empty dataset. * +1 for successful completion K number of clusters C array[K,NFeatures], rows of the array store centers CIdx array[NPoints], which contains cluster indexes IterationsCount actual number of iterations performed by clusterizer. If algorithm performed more than one random restart, total number of iterations is returned. Energy merit function, "energy", sum of squared deviations from cluster centers -- ALGLIB -- Copyright 27.11.2012 by Bochkanov Sergey *************************************************************************/ public class kmeansreport : apobject { public int npoints; public int nfeatures; public int terminationtype; public int iterationscount; public double energy; public int k; public double[,] c; public int[] cidx; public kmeansreport() { init(); } public override void init() { c = new double[0,0]; cidx = new int[0]; } public override alglib.apobject make_copy() { kmeansreport _result = new kmeansreport(); _result.npoints = npoints; _result.nfeatures = nfeatures; _result.terminationtype = terminationtype; _result.iterationscount = iterationscount; _result.energy = energy; _result.k = k; _result.c = (double[,])c.Clone(); _result.cidx = (int[])cidx.Clone(); return _result; } }; public const int kmeansblocksize = 32; public const int kmeansparalleldim = 8; public const int kmeansparallelk = 4; public const double complexitymultiplier = 1.0; /************************************************************************* This function initializes clusterizer object. Newly initialized object is empty, i.e. it does not contain dataset. You should use it as follows: 1. creation 2. dataset is added with ClusterizerSetPoints() 3. additional parameters are set 3. clusterization is performed with one of the clustering functions -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizercreate(clusterizerstate s, alglib.xparams _params) { s.npoints = 0; s.nfeatures = 0; s.disttype = 2; s.ahcalgo = 0; s.kmeansrestarts = 1; s.kmeansmaxits = 0; s.kmeansinitalgo = 0; s.kmeansdbgnoits = false; s.seed = 1; kmeansinitbuf(s.kmeanstmp, _params); } /************************************************************************* This function adds dataset to the clusterizer structure. This function overrides all previous calls of ClusterizerSetPoints() or ClusterizerSetDistances(). INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() XY - array[NPoints,NFeatures], dataset NPoints - number of points, >=0 NFeatures- number of features, >=1 DistType- distance function: * 0 Chebyshev distance (L-inf norm) * 1 city block distance (L1 norm) * 2 Euclidean distance (L2 norm), non-squared * 10 Pearson correlation: dist(a,b) = 1-corr(a,b) * 11 Absolute Pearson correlation: dist(a,b) = 1-|corr(a,b)| * 12 Uncentered Pearson correlation (cosine of the angle): dist(a,b) = a'*b/(|a|*|b|) * 13 Absolute uncentered Pearson correlation dist(a,b) = |a'*b|/(|a|*|b|) * 20 Spearman rank correlation: dist(a,b) = 1-rankcorr(a,b) * 21 Absolute Spearman rank correlation dist(a,b) = 1-|rankcorr(a,b)| NOTE 1: different distance functions have different performance penalty: * Euclidean or Pearson correlation distances are the fastest ones * Spearman correlation distance function is a bit slower * city block and Chebyshev distances are order of magnitude slower The reason behing difference in performance is that correlation-based distance functions are computed using optimized linear algebra kernels, while Chebyshev and city block distance functions are computed using simple nested loops with two branches at each iteration. NOTE 2: different clustering algorithms have different limitations: * agglomerative hierarchical clustering algorithms may be used with any kind of distance metric * k-means++ clustering algorithm may be used only with Euclidean distance function Thus, list of specific clustering algorithms you may use depends on distance function you specify when you set your dataset. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetpoints(clusterizerstate s, double[,] xy, int npoints, int nfeatures, int disttype, alglib.xparams _params) { int i = 0; int i_ = 0; alglib.ap.assert((((((((disttype==0 || disttype==1) || disttype==2) || disttype==10) || disttype==11) || disttype==12) || disttype==13) || disttype==20) || disttype==21, "ClusterizerSetPoints: incorrect DistType"); alglib.ap.assert(npoints>=0, "ClusterizerSetPoints: NPoints<0"); alglib.ap.assert(nfeatures>=1, "ClusterizerSetPoints: NFeatures<1"); alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "ClusterizerSetPoints: Rows(XY)=nfeatures, "ClusterizerSetPoints: Cols(XY)=0, "ClusterizerSetDistances: NPoints<0"); alglib.ap.assert(alglib.ap.rows(d)>=npoints, "ClusterizerSetDistances: Rows(D)=npoints, "ClusterizerSetDistances: Cols(D)=(double)(0), "ClusterizerSetDistances: D contains infinite, NAN or negative elements"); s.d[i,j] = d[i,j]; s.d[j,i] = d[i,j]; } s.d[i,i] = 0; } } /************************************************************************* This function sets agglomerative hierarchical clustering algorithm INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() Algo - algorithm type: * 0 complete linkage (default algorithm) * 1 single linkage * 2 unweighted average linkage * 3 weighted average linkage * 4 Ward's method NOTE: Ward's method works correctly only with Euclidean distance, that's why algorithm will return negative termination code (failure) for any other distance type. It is possible, however, to use this method with user-supplied distance matrix. It is your responsibility to pass one which was calculated with Euclidean distance function. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetahcalgo(clusterizerstate s, int algo, alglib.xparams _params) { alglib.ap.assert((((algo==0 || algo==1) || algo==2) || algo==3) || algo==4, "ClusterizerSetHCAlgo: incorrect algorithm type"); s.ahcalgo = algo; } /************************************************************************* This function sets k-means properties: number of restarts and maximum number of iterations per one run. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() Restarts- restarts count, >=1. k-means++ algorithm performs several restarts and chooses best set of centers (one with minimum squared distance). MaxIts - maximum number of k-means iterations performed during one run. >=0, zero value means that algorithm performs unlimited number of iterations. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetkmeanslimits(clusterizerstate s, int restarts, int maxits, alglib.xparams _params) { alglib.ap.assert(restarts>=1, "ClusterizerSetKMeansLimits: Restarts<=0"); alglib.ap.assert(maxits>=0, "ClusterizerSetKMeansLimits: MaxIts<0"); s.kmeansrestarts = restarts; s.kmeansmaxits = maxits; } /************************************************************************* This function sets k-means initialization algorithm. Several different algorithms can be chosen, including k-means++. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() InitAlgo- initialization algorithm: * 0 automatic selection ( different versions of ALGLIB may select different algorithms) * 1 random initialization * 2 k-means++ initialization (best quality of initial centers, but long non-parallelizable initialization phase with bad cache locality) * 3 "fast-greedy" algorithm with efficient, easy to parallelize initialization. Quality of initial centers is somewhat worse than that of k-means++. This algorithm is a default one in the current version of ALGLIB. *-1 "debug" algorithm which always selects first K rows of dataset; this algorithm is used for debug purposes only. Do not use it in the industrial code! -- ALGLIB -- Copyright 21.01.2015 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetkmeansinit(clusterizerstate s, int initalgo, alglib.xparams _params) { alglib.ap.assert(initalgo>=-1 && initalgo<=3, "ClusterizerSetKMeansInit: InitAlgo is incorrect"); s.kmeansinitalgo = initalgo; } /************************************************************************* This function sets seed which is used to initialize internal RNG. By default, deterministic seed is used - same for each run of clusterizer. If you specify non-deterministic seed value, then some algorithms which depend on random initialization (in current version: k-means) may return slightly different results after each run. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() Seed - seed: * positive values = use deterministic seed for each run of algorithms which depend on random initialization * zero or negative values = use non-deterministic seed -- ALGLIB -- Copyright 08.06.2017 by Bochkanov Sergey *************************************************************************/ public static void clusterizersetseed(clusterizerstate s, int seed, alglib.xparams _params) { s.seed = seed; } /************************************************************************* This function performs agglomerative hierarchical clustering ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. NOTE: Agglomerative hierarchical clustering algorithm has two phases: distance matrix calculation and clustering itself. Only first phase (distance matrix calculation) is accelerated by Intel MKL and multithreading. Thus, acceleration is significant only for medium or high-dimensional problems. Although activating multithreading gives some speedup over single- threaded execution, you should not expect nearly-linear scaling with respect to cores count. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() OUTPUT PARAMETERS: Rep - clustering results; see description of AHCReport structure for more information. NOTE 1: hierarchical clustering algorithms require large amounts of memory. In particular, this implementation needs sizeof(double)*NPoints^2 bytes, which are used to store distance matrix. In case we work with user-supplied matrix, this amount is multiplied by 2 (we have to store original matrix and to work with its copy). For example, problem with 10000 points would require 800M of RAM, even when working in a 1-dimensional space. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizerrunahc(clusterizerstate s, ahcreport rep, alglib.xparams _params) { int npoints = 0; int nfeatures = 0; npoints = s.npoints; nfeatures = s.nfeatures; // // Fill Rep.NPoints, quick exit when NPoints<=1 // rep.npoints = npoints; if( npoints==0 ) { rep.p = new int[0]; rep.z = new int[0, 0]; rep.pz = new int[0, 0]; rep.pm = new int[0, 0]; rep.mergedist = new double[0]; rep.terminationtype = 1; return; } if( npoints==1 ) { rep.p = new int[1]; rep.z = new int[0, 0]; rep.pz = new int[0, 0]; rep.pm = new int[0, 0]; rep.mergedist = new double[0]; rep.p[0] = 0; rep.terminationtype = 1; return; } // // More than one point // if( s.disttype==-1 ) { // // Run clusterizer with user-supplied distance matrix // clusterizerrunahcinternal(s, ref s.d, rep, _params); return; } else { // // Check combination of AHC algo and distance type // if( s.ahcalgo==4 && s.disttype!=2 ) { rep.terminationtype = -5; return; } // // Build distance matrix D. // clusterizergetdistancesbuf(s.distbuf, s.xy, npoints, nfeatures, s.disttype, ref s.tmpd, _params); // // Run clusterizer // clusterizerrunahcinternal(s, ref s.tmpd, rep, _params); return; } } /************************************************************************* This function performs clustering by k-means++ algorithm. You may change algorithm properties by calling: * ClusterizerSetKMeansLimits() to change number of restarts or iterations * ClusterizerSetKMeansInit() to change initialization algorithm By default, one restart and unlimited number of iterations are used. Initialization algorithm is chosen automatically. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. NOTE: k-means clustering algorithm has two phases: selection of initial centers and clustering itself. ALGLIB parallelizes both phases. Parallel version is optimized for the following scenario: medium or high-dimensional problem (8 or more dimensions) with large number of points and clusters. However, some speed-up can be obtained even when assumptions above are violated. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() K - number of clusters, K>=0. K can be zero only when algorithm is called for empty dataset, in this case completion code is set to success (+1). If K=0 and dataset size is non-zero, we can not meaningfully assign points to some center (there are no centers because K=0) and return -3 as completion code (failure). OUTPUT PARAMETERS: Rep - clustering results; see description of KMeansReport structure for more information. NOTE 1: k-means clustering can be performed only for datasets with Euclidean distance function. Algorithm will return negative completion code in Rep.TerminationType in case dataset was added to clusterizer with DistType other than Euclidean (or dataset was specified by distance matrix instead of explicitly given points). NOTE 2: by default, k-means uses non-deterministic seed to initialize RNG which is used to select initial centers. As result, each run of algorithm may return different values. If you need deterministic behavior, use ClusterizerSetSeed() function. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizerrunkmeans(clusterizerstate s, int k, kmeansreport rep, alglib.xparams _params) { double[,] dummy = new double[0,0]; alglib.ap.assert(k>=0, "ClusterizerRunKMeans: K<0"); // // Incorrect distance type // if( s.disttype!=2 ) { rep.npoints = s.npoints; rep.terminationtype = -5; rep.k = k; rep.iterationscount = 0; rep.energy = 0.0; return; } // // K>NPoints or (K=0 and NPoints>0) // if( k>s.npoints || (k==0 && s.npoints>0) ) { rep.npoints = s.npoints; rep.terminationtype = -3; rep.k = k; rep.iterationscount = 0; rep.energy = 0.0; return; } // // No points // if( s.npoints==0 ) { rep.npoints = 0; rep.terminationtype = 1; rep.k = k; rep.iterationscount = 0; rep.energy = 0.0; return; } // // Normal case: // 1<=K<=NPoints, Euclidean distance // rep.npoints = s.npoints; rep.nfeatures = s.nfeatures; rep.k = k; rep.npoints = s.npoints; rep.nfeatures = s.nfeatures; kmeansgenerateinternal(s.xy, s.npoints, s.nfeatures, k, s.kmeansinitalgo, s.seed, s.kmeansmaxits, s.kmeansrestarts, s.kmeansdbgnoits, ref rep.terminationtype, ref rep.iterationscount, ref dummy, false, ref rep.c, true, ref rep.cidx, ref rep.energy, s.kmeanstmp, _params); } /************************************************************************* This function returns distance matrix for dataset ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! * hardware vendor (Intel) implementations of linear algebra primitives ! (C++ and C# versions, x86/x64 platform) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: XY - array[NPoints,NFeatures], dataset NPoints - number of points, >=0 NFeatures- number of features, >=1 DistType- distance function: * 0 Chebyshev distance (L-inf norm) * 1 city block distance (L1 norm) * 2 Euclidean distance (L2 norm, non-squared) * 10 Pearson correlation: dist(a,b) = 1-corr(a,b) * 11 Absolute Pearson correlation: dist(a,b) = 1-|corr(a,b)| * 12 Uncentered Pearson correlation (cosine of the angle): dist(a,b) = a'*b/(|a|*|b|) * 13 Absolute uncentered Pearson correlation dist(a,b) = |a'*b|/(|a|*|b|) * 20 Spearman rank correlation: dist(a,b) = 1-rankcorr(a,b) * 21 Absolute Spearman rank correlation dist(a,b) = 1-|rankcorr(a,b)| OUTPUT PARAMETERS: D - array[NPoints,NPoints], distance matrix (full matrix is returned, with lower and upper triangles) NOTE: different distance functions have different performance penalty: * Euclidean or Pearson correlation distances are the fastest ones * Spearman correlation distance function is a bit slower * city block and Chebyshev distances are order of magnitude slower The reason behing difference in performance is that correlation-based distance functions are computed using optimized linear algebra kernels, while Chebyshev and city block distance functions are computed using simple nested loops with two branches at each iteration. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ public static void clusterizergetdistances(double[,] xy, int npoints, int nfeatures, int disttype, ref double[,] d, alglib.xparams _params) { apserv.apbuffers buf = new apserv.apbuffers(); d = new double[0,0]; alglib.ap.assert(nfeatures>=1, "ClusterizerGetDistances: NFeatures<1"); alglib.ap.assert(npoints>=0, "ClusterizerGetDistances: NPoints<1"); alglib.ap.assert((((((((disttype==0 || disttype==1) || disttype==2) || disttype==10) || disttype==11) || disttype==12) || disttype==13) || disttype==20) || disttype==21, "ClusterizerGetDistances: incorrect DistType"); alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "ClusterizerGetDistances: Rows(XY)=nfeatures, "ClusterizerGetDistances: Cols(XY)=1, "ClusterizerGetDistancesBuf: NFeatures<1"); alglib.ap.assert(npoints>=0, "ClusterizerGetDistancesBuf: NPoints<1"); alglib.ap.assert((((((((disttype==0 || disttype==1) || disttype==2) || disttype==10) || disttype==11) || disttype==12) || disttype==13) || disttype==20) || disttype==21, "ClusterizerGetDistancesBuf: incorrect DistType"); alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "ClusterizerGetDistancesBuf: Rows(XY)=nfeatures, "ClusterizerGetDistancesBuf: Cols(XY)(double)(0) ) { buf.ra0[i] = 1/Math.Sqrt(d[i,i]); } else { buf.ra0[i] = 0.0; } } for(i=0; i<=npoints-1; i++) { v = buf.ra0[i]; d[i,i] = 0.0; for(j=i+1; j<=npoints-1; j++) { vv = d[i,j]*v*buf.ra0[j]; if( disttype==20 ) { vr = 1-vv; } else { vr = 1-Math.Abs(vv); } if( (double)(vr)<(double)(0) ) { vr = 0.0; } d[i,j] = vr; } } ablas.rmatrixenforcesymmetricity(d, npoints, true, _params); return; } alglib.ap.assert(false); } /************************************************************************* This function takes as input clusterization report Rep, desired clusters count K, and builds top K clusters from hierarchical clusterization tree. It returns assignment of points to clusters (array of cluster indexes). INPUT PARAMETERS: Rep - report from ClusterizerRunAHC() performed on XY K - desired number of clusters, 1<=K<=NPoints. K can be zero only when NPoints=0. OUTPUT PARAMETERS: CIdx - array[NPoints], I-th element contains cluster index (from 0 to K-1) for I-th point of the dataset. CZ - array[K]. This array allows to convert cluster indexes returned by this function to indexes used by Rep.Z. J-th cluster returned by this function corresponds to CZ[J]-th cluster stored in Rep.Z/PZ/PM. It is guaranteed that CZ[I]=0, "ClusterizerGetKClusters: internal error in Rep integrity"); alglib.ap.assert(k>=0, "ClusterizerGetKClusters: K<=0"); alglib.ap.assert(k<=npoints, "ClusterizerGetKClusters: K>NPoints"); alglib.ap.assert(k>0 || npoints==0, "ClusterizerGetKClusters: K<=0"); alglib.ap.assert(npoints==rep.npoints, "ClusterizerGetKClusters: NPoints<>Rep.NPoints"); // // Quick exit // if( npoints==0 ) { return; } if( npoints==1 ) { cz = new int[1]; cidx = new int[1]; cz[0] = 0; cidx[0] = 0; return; } // // Replay merges, from top to bottom, // keep track of clusters being present at the moment // presentclusters = new bool[2*npoints-1]; tmpidx = new int[npoints]; for(i=0; i<=2*npoints-3; i++) { presentclusters[i] = false; } presentclusters[2*npoints-2] = true; for(i=0; i<=npoints-1; i++) { tmpidx[i] = 2*npoints-2; } for(mergeidx=npoints-2; mergeidx>=npoints-k; mergeidx--) { // // Update information about clusters being present at the moment // presentclusters[npoints+mergeidx] = false; presentclusters[rep.z[mergeidx,0]] = true; presentclusters[rep.z[mergeidx,1]] = true; // // Update TmpIdx according to the current state of the dataset // // NOTE: TmpIdx contains cluster indexes from [0..2*NPoints-2]; // we will convert them to [0..K-1] later. // i0 = rep.pm[mergeidx,0]; i1 = rep.pm[mergeidx,1]; t = rep.z[mergeidx,0]; for(i=i0; i<=i1; i++) { tmpidx[i] = t; } i0 = rep.pm[mergeidx,2]; i1 = rep.pm[mergeidx,3]; t = rep.z[mergeidx,1]; for(i=i0; i<=i1; i++) { tmpidx[i] = t; } } // // Fill CZ - array which allows us to convert cluster indexes // from one system to another. // cz = new int[k]; clusterindexes = new int[2*npoints-1]; t = 0; for(i=0; i<=2*npoints-2; i++) { if( presentclusters[i] ) { cz[t] = i; clusterindexes[i] = t; t = t+1; } } alglib.ap.assert(t==k, "ClusterizerGetKClusters: internal error"); // // Convert indexes stored in CIdx // cidx = new int[npoints]; for(i=0; i<=npoints-1; i++) { cidx[i] = clusterindexes[tmpidx[rep.p[i]]]; } } /************************************************************************* This function accepts AHC report Rep, desired minimum intercluster distance and returns top clusters from hierarchical clusterization tree which are separated by distance R or HIGHER. It returns assignment of points to clusters (array of cluster indexes). There is one more function with similar name - ClusterizerSeparatedByCorr, which returns clusters with intercluster correlation equal to R or LOWER (note: higher for distance, lower for correlation). INPUT PARAMETERS: Rep - report from ClusterizerRunAHC() performed on XY R - desired minimum intercluster distance, R>=0 OUTPUT PARAMETERS: K - number of clusters, 1<=K<=NPoints CIdx - array[NPoints], I-th element contains cluster index (from 0 to K-1) for I-th point of the dataset. CZ - array[K]. This array allows to convert cluster indexes returned by this function to indexes used by Rep.Z. J-th cluster returned by this function corresponds to CZ[J]-th cluster stored in Rep.Z/PZ/PM. It is guaranteed that CZ[I]=(double)(0), "ClusterizerSeparatedByDist: R is infinite or less than 0"); k = 1; while( k=(double)(r) ) { k = k+1; } clusterizergetkclusters(rep, k, ref cidx, ref cz, _params); } /************************************************************************* This function accepts AHC report Rep, desired maximum intercluster correlation and returns top clusters from hierarchical clusterization tree which are separated by correlation R or LOWER. It returns assignment of points to clusters (array of cluster indexes). There is one more function with similar name - ClusterizerSeparatedByDist, which returns clusters with intercluster distance equal to R or HIGHER (note: higher for distance, lower for correlation). INPUT PARAMETERS: Rep - report from ClusterizerRunAHC() performed on XY R - desired maximum intercluster correlation, -1<=R<=+1 OUTPUT PARAMETERS: K - number of clusters, 1<=K<=NPoints CIdx - array[NPoints], I-th element contains cluster index (from 0 to K-1) for I-th point of the dataset. CZ - array[K]. This array allows to convert cluster indexes returned by this function to indexes used by Rep.Z. J-th cluster returned by this function corresponds to CZ[J]-th cluster stored in Rep.Z/PZ/PM. It is guaranteed that CZ[I]=(double)(-1)) && (double)(r)<=(double)(1), "ClusterizerSeparatedByCorr: R is infinite or less than 0"); k = 1; while( k=(double)(1-r) ) { k = k+1; } clusterizergetkclusters(rep, k, ref cidx, ref cz, _params); } /************************************************************************* K-means++ initialization INPUT PARAMETERS: Buf - special reusable structure which stores previously allocated memory, intended to avoid memory fragmentation when solving multiple subsequent problems. Must be initialized prior to usage. OUTPUT PARAMETERS: Buf - initialized structure -- ALGLIB -- Copyright 24.07.2015 by Bochkanov Sergey *************************************************************************/ public static void kmeansinitbuf(kmeansbuffers buf, alglib.xparams _params) { apserv.apbuffers updateseed = new apserv.apbuffers(); alglib.smp.ae_shared_pool_set_seed(buf.updatepool, updateseed); } /************************************************************************* K-means++ clusterization INPUT PARAMETERS: XY - dataset, array [0..NPoints-1,0..NVars-1]. NPoints - dataset size, NPoints>=K NVars - number of variables, NVars>=1 K - desired number of clusters, K>=1 InitAlgo - initialization algorithm: * 0 - automatic selection of best algorithm * 1 - random selection of centers * 2 - k-means++ * 3 - fast-greedy init *-1 - first K rows of dataset are used (special debug algorithm) Seed - seed value for internal RNG: * positive value is used to initialize RNG in order to induce deterministic behavior of algorithm * zero or negative value means that random seed is generated MaxIts - iterations limit or zero for no limit Restarts - number of restarts, Restarts>=1 KMeansDbgNoIts- debug flag; if set, Lloyd's iteration is not performed, only initialization phase. Buf - special reusable structure which stores previously allocated memory, intended to avoid memory fragmentation when solving multiple subsequent problems: * MUST BE INITIALIZED WITH KMeansInitBuffers() CALL BEFORE FIRST PASS TO THIS FUNCTION! * subsequent passes must be made without re-initialization OUTPUT PARAMETERS: Info - return code: * -3, if task is degenerate (number of distinct points is less than K) * -1, if incorrect NPoints/NFeatures/K/Restarts was passed * 1, if subroutine finished successfully IterationsCount- actual number of iterations performed by clusterizer CCol - array[0..NVars-1,0..K-1].matrix whose columns store cluster's centers NeedCCol - True in case caller requires to store result in CCol CRow - array[0..K-1,0..NVars-1], same as CCol, but centers are stored in rows NeedCRow - True in case caller requires to store result in CCol XYC - array[NPoints], which contains cluster indexes Energy - merit function of clusterization -- ALGLIB -- Copyright 21.03.2009 by Bochkanov Sergey *************************************************************************/ public static void kmeansgenerateinternal(double[,] xy, int npoints, int nvars, int k, int initalgo, int seed, int maxits, int restarts, bool kmeansdbgnoits, ref int info, ref int iterationscount, ref double[,] ccol, bool needccol, ref double[,] crow, bool needcrow, ref int[] xyc, ref double energy, kmeansbuffers buf, alglib.xparams _params) { int i = 0; int j = 0; int i1 = 0; double e = 0; double eprev = 0; double v = 0; double vv = 0; bool waschanges = new bool(); bool zerosizeclusters = new bool(); int pass = 0; int itcnt = 0; hqrnd.hqrndstate rs = new hqrnd.hqrndstate(); int i_ = 0; info = 0; iterationscount = 0; ccol = new double[0,0]; crow = new double[0,0]; xyc = new int[0]; energy = 0; // // Test parameters // if( ((npoints=(double)(eprev) ) { break; } // // Update EPrev // eprev = e; } } else { // // Debug mode: no Lloyd's iteration. // We just calculate potential E. // kmeansupdatedistances(xy, 0, npoints, nvars, buf.ct, 0, k, xyc, buf.d2, buf.updatepool, _params); e = 0; for(i=0; i<=npoints-1; i++) { e = e+buf.d2[i]; } } // // Compare E with best centers found so far // if( (double)(e)<(double)(energy) ) { // // store partition. // energy = e; blas.copymatrix(buf.ct, 0, k-1, 0, nvars-1, ref buf.ctbest, 0, k-1, 0, nvars-1, _params); for(i=0; i<=npoints-1; i++) { buf.xycbest[i] = xyc[i]; } } } // // Copy and transpose // if( needccol ) { ccol = new double[nvars, k]; blas.copyandtranspose(buf.ctbest, 0, k-1, 0, nvars-1, ref ccol, 0, nvars-1, 0, k-1, _params); } if( needcrow ) { crow = new double[k, nvars]; ablas.rmatrixcopy(k, nvars, buf.ctbest, 0, 0, crow, 0, 0, _params); } for(i=0; i<=npoints-1; i++) { xyc[i] = buf.xycbest[i]; } } /************************************************************************* This procedure recalculates distances from points to centers and assigns each point to closest center. INPUT PARAMETERS: XY - dataset, array [0..NPoints-1,0..NVars-1]. Idx0,Idx1 - define range of dataset [Idx0,Idx1) to process; right boundary is not included. NVars - number of variables, NVars>=1 CT - matrix of centers, centers are stored in rows CIdx0,CIdx1 - define range of centers [CIdx0,CIdx1) to process; right boundary is not included. XYC - preallocated output buffer, XYDist2 - preallocated output buffer Tmp - temporary buffer, automatically reallocated if needed BufferPool - shared pool seeded with instance of APBuffers structure (seed instance can be unitialized). It is recommended to use this pool only with KMeansUpdateDistances() function. OUTPUT PARAMETERS: XYC - new assignment of points to centers are stored in [Idx0,Idx1) XYDist2 - squared distances from points to their centers are stored in [Idx0,Idx1) -- ALGLIB -- Copyright 21.01.2015 by Bochkanov Sergey *************************************************************************/ public static void kmeansupdatedistances(double[,] xy, int idx0, int idx1, int nvars, double[,] ct, int cidx0, int cidx1, int[] xyc, double[] xydist2, alglib.smp.shared_pool bufferpool, alglib.xparams _params) { int i = 0; int i0 = 0; int i1 = 0; int j = 0; int cclosest = 0; double dclosest = 0; double vv = 0; apserv.apbuffers buf = null; double rcomplexity = 0; int task0 = 0; int task1 = 0; int pblkcnt = 0; int cblkcnt = 0; int vblkcnt = 0; int pblk = 0; int cblk = 0; int vblk = 0; int p0 = 0; int p1 = 0; int c0 = 0; int c1 = 0; int v0 = 0; int v1 = 0; double v00 = 0; double v01 = 0; double v10 = 0; double v11 = 0; double vp0 = 0; double vp1 = 0; double vc0 = 0; double vc1 = 0; int pcnt = 0; int pcntpadded = 0; int ccnt = 0; int ccntpadded = 0; int offs0 = 0; int offs00 = 0; int offs01 = 0; int offs10 = 0; int offs11 = 0; int vcnt = 0; int stride = 0; // // Quick exit for special cases // if( idx1<=idx0 ) { return; } if( cidx1<=cidx0 ) { return; } if( nvars<=0 ) { return; } // // Try to recursively divide/process dataset // // NOTE: real arithmetics is used to avoid integer overflow on large problem sizes // rcomplexity = 2*apserv.rmul3(idx1-idx0, cidx1-cidx0, nvars, _params); if( (double)(rcomplexity)>=(double)(apserv.smpactivationlevel(_params)) && idx1-idx0>=2*kmeansblocksize ) { if( _trypexec_kmeansupdatedistances(xy,idx0,idx1,nvars,ct,cidx0,cidx1,xyc,xydist2,bufferpool, _params) ) { return; } } if( (((double)(rcomplexity)>=(double)(apserv.spawnlevel(_params)) && idx1-idx0>=2*kmeansblocksize) && nvars>=kmeansparalleldim) && cidx1-cidx0>=kmeansparallelk ) { apserv.splitlength(idx1-idx0, kmeansblocksize, ref task0, ref task1, _params); kmeansupdatedistances(xy, idx0, idx0+task0, nvars, ct, cidx0, cidx1, xyc, xydist2, bufferpool, _params); kmeansupdatedistances(xy, idx0+task0, idx1, nvars, ct, cidx0, cidx1, xyc, xydist2, bufferpool, _params); return; } // // Dataset chunk is selected. // // Process it with blocked algorithm: // * iterate over points, process them in KMeansBlockSize-ed chunks // * for each chunk of dataset, iterate over centers, process them in KMeansBlockSize-ed chunks // * for each chunk of dataset/centerset, iterate over variables, process them in KMeansBlockSize-ed chunks // alglib.ap.assert(kmeansblocksize%2==0, "KMeansUpdateDistances: internal error"); alglib.smp.ae_shared_pool_retrieve(bufferpool, ref buf); apserv.rvectorsetlengthatleast(ref buf.ra0, kmeansblocksize*kmeansblocksize, _params); apserv.rvectorsetlengthatleast(ref buf.ra1, kmeansblocksize*kmeansblocksize, _params); apserv.rvectorsetlengthatleast(ref buf.ra2, kmeansblocksize*kmeansblocksize, _params); apserv.rvectorsetlengthatleast(ref buf.ra3, kmeansblocksize, _params); apserv.ivectorsetlengthatleast(ref buf.ia3, kmeansblocksize, _params); pblkcnt = apserv.chunkscount(idx1-idx0, kmeansblocksize, _params); cblkcnt = apserv.chunkscount(cidx1-cidx0, kmeansblocksize, _params); vblkcnt = apserv.chunkscount(nvars, kmeansblocksize, _params); for(pblk=0; pblk<=pblkcnt-1; pblk++) { // // Process PBlk-th chunk of dataset. // p0 = idx0+pblk*kmeansblocksize; p1 = Math.Min(p0+kmeansblocksize, idx1); // // Prepare RA3[]/IA3[] for storage of best distances and best cluster numbers. // for(i=0; i<=kmeansblocksize-1; i++) { buf.ra3[i] = math.maxrealnumber; buf.ia3[i] = -1; } // // Iterare over chunks of centerset. // for(cblk=0; cblk<=cblkcnt-1; cblk++) { // // Process CBlk-th chunk of centerset // c0 = cidx0+cblk*kmeansblocksize; c1 = Math.Min(c0+kmeansblocksize, cidx1); // // At this point we have to calculate a set of pairwise distances // between points [P0,P1) and centers [C0,C1) and select best center // for each point. It can also be done with blocked algorithm // (blocking for variables). // // Following arrays are used: // * RA0[] - matrix of distances, padded by zeros for even size, // rows are stored with stride KMeansBlockSize. // * RA1[] - matrix of points (variables corresponding to current // block are extracted), padded by zeros for even size, // rows are stored with stride KMeansBlockSize. // * RA2[] - matrix of centers (variables corresponding to current // block are extracted), padded by zeros for even size, // rows are stored with stride KMeansBlockSize. // // pcnt = p1-p0; pcntpadded = pcnt+pcnt%2; ccnt = c1-c0; ccntpadded = ccnt+ccnt%2; stride = kmeansblocksize; alglib.ap.assert(pcntpadded<=kmeansblocksize, "KMeansUpdateDistances: integrity error"); alglib.ap.assert(ccntpadded<=kmeansblocksize, "KMeansUpdateDistances: integrity error"); for(i=0; i<=pcntpadded-1; i++) { for(j=0; j<=ccntpadded-1; j++) { buf.ra0[i*stride+j] = 0.0; } } for(vblk=0; vblk<=vblkcnt-1; vblk++) { // // Fetch VBlk-th block of variables to arrays RA1 (points) and RA2 (centers). // Pad points and centers with zeros. // v0 = vblk*kmeansblocksize; v1 = Math.Min(v0+kmeansblocksize, nvars); vcnt = v1-v0; for(i=0; i<=pcnt-1; i++) { for(j=0; j<=vcnt-1; j++) { buf.ra1[i*stride+j] = xy[p0+i,v0+j]; } } for(i=pcnt; i<=pcntpadded-1; i++) { for(j=0; j<=vcnt-1; j++) { buf.ra1[i*stride+j] = 0.0; } } for(i=0; i<=ccnt-1; i++) { for(j=0; j<=vcnt-1; j++) { buf.ra2[i*stride+j] = ct[c0+i,v0+j]; } } for(i=ccnt; i<=ccntpadded-1; i++) { for(j=0; j<=vcnt-1; j++) { buf.ra2[i*stride+j] = 0.0; } } // // Update distance matrix with sums-of-squared-differences of RA1 and RA2 // i0 = 0; while( i0=1 InitAlgo - initialization algorithm: * 0 - automatic selection of best algorithm * 1 - random selection * 2 - k-means++ * 3 - fast-greedy init *-1 - first K rows of dataset are used (debug algorithm) RS - RNG used to select centers K - number of centers, K>=1 CT - possibly preallocated output buffer, resized if needed InitBuf - internal buffer, possibly unitialized instance of APBuffers. It is recommended to use this instance only with SelectInitialCenters() and FixCenters() functions, because these functions may allocate really large storage. UpdatePool - shared pool seeded with instance of APBuffers structure (seed instance can be unitialized). Used internally with KMeansUpdateDistances() function. It is recommended to use this pool ONLY with KMeansUpdateDistances() function. OUTPUT PARAMETERS: CT - set of K clusters, one per row RESULT: True on success, False on failure (impossible to create K independent clusters) -- ALGLIB -- Copyright 21.01.2015 by Bochkanov Sergey *************************************************************************/ private static void selectinitialcenters(double[,] xy, int npoints, int nvars, int initalgo, hqrnd.hqrndstate rs, int k, ref double[,] ct, apserv.apbuffers initbuf, alglib.smp.shared_pool updatepool, alglib.xparams _params) { int cidx = 0; int i = 0; int j = 0; double v = 0; double vv = 0; double s = 0; int lastnz = 0; int ptidx = 0; int samplesize = 0; int samplescntnew = 0; int samplescntall = 0; double samplescale = 0; int i_ = 0; // // Check parameters // alglib.ap.assert(npoints>0, "SelectInitialCenters: internal error"); alglib.ap.assert(nvars>0, "SelectInitialCenters: internal error"); alglib.ap.assert(k>0, "SelectInitialCenters: internal error"); if( initalgo==0 ) { initalgo = 3; } apserv.rmatrixsetlengthatleast(ref ct, k, nvars, _params); // // Random initialization // if( initalgo==-1 ) { for(i=0; i<=k-1; i++) { for(i_=0; i_<=nvars-1;i_++) { ct[i,i_] = xy[i%npoints,i_]; } } return; } // // Random initialization // if( initalgo==1 ) { for(i=0; i<=k-1; i++) { j = hqrnd.hqrnduniformi(rs, npoints, _params); for(i_=0; i_<=nvars-1;i_++) { ct[i,i_] = xy[j,i_]; } } return; } // // k-means++ initialization // if( initalgo==2 ) { // // Prepare distances array. // Select initial center at random. // apserv.rvectorsetlengthatleast(ref initbuf.ra0, npoints, _params); for(i=0; i<=npoints-1; i++) { initbuf.ra0[i] = math.maxrealnumber; } ptidx = hqrnd.hqrnduniformi(rs, npoints, _params); for(i_=0; i_<=nvars-1;i_++) { ct[0,i_] = xy[ptidx,i_]; } // // For each newly added center repeat: // * reevaluate distances from points to best centers // * sample points with probability dependent on distance // * add new center // for(cidx=0; cidx<=k-2; cidx++) { // // Reevaluate distances // s = 0.0; for(i=0; i<=npoints-1; i++) { v = 0.0; for(j=0; j<=nvars-1; j++) { vv = xy[i,j]-ct[cidx,j]; v = v+vv*vv; } if( (double)(v)<(double)(initbuf.ra0[i]) ) { initbuf.ra0[i] = v; } s = s+initbuf.ra0[i]; } // // If all distances are zero, it means that we can not find enough // distinct points. In this case we just select non-distinct center // at random and continue iterations. This issue will be handled // later in the FixCenters() function. // if( (double)(s)==(double)(0.0) ) { ptidx = hqrnd.hqrnduniformi(rs, npoints, _params); for(i_=0; i_<=nvars-1;i_++) { ct[cidx+1,i_] = xy[ptidx,i_]; } continue; } // // Select point as center using its distance. // We also handle situation when because of rounding errors // no point was selected - in this case, last non-zero one // will be used. // v = hqrnd.hqrnduniformr(rs, _params); vv = 0.0; lastnz = -1; ptidx = -1; for(i=0; i<=npoints-1; i++) { if( (double)(initbuf.ra0[i])==(double)(0.0) ) { continue; } lastnz = i; vv = vv+initbuf.ra0[i]; if( (double)(v)<=(double)(vv/s) ) { ptidx = i; break; } } alglib.ap.assert(lastnz>=0, "SelectInitialCenters: integrity error"); if( ptidx<0 ) { ptidx = lastnz; } for(i_=0; i_<=nvars-1;i_++) { ct[cidx+1,i_] = xy[ptidx,i_]; } } return; } // // "Fast-greedy" algorithm based on "Scalable k-means++". // // We perform several rounds, within each round we sample about 0.5*K points // (not exactly 0.5*K) until we have 2*K points sampled. Before each round // we calculate distances from dataset points to closest points sampled so far. // We sample dataset points independently using distance xtimes 0.5*K divided by total // as probability (similar to k-means++, but each point is sampled independently; // after each round we have roughtly 0.5*K points added to sample). // // After sampling is done, we run "greedy" version of k-means++ on this subsample // which selects most distant point on every round. // if( initalgo==3 ) { // // Prepare arrays. // Select initial center at random, add it to "new" part of sample, // which is stored at the beginning of the array // samplesize = 2*k; samplescale = 0.5*k; apserv.rmatrixsetlengthatleast(ref initbuf.rm0, samplesize, nvars, _params); ptidx = hqrnd.hqrnduniformi(rs, npoints, _params); for(i_=0; i_<=nvars-1;i_++) { initbuf.rm0[0,i_] = xy[ptidx,i_]; } samplescntnew = 1; samplescntall = 1; apserv.rvectorsetlengthatleast(ref initbuf.ra0, npoints, _params); apserv.rvectorsetlengthatleast(ref initbuf.ra1, npoints, _params); apserv.ivectorsetlengthatleast(ref initbuf.ia1, npoints, _params); for(i=0; i<=npoints-1; i++) { initbuf.ra0[i] = math.maxrealnumber; } // // Repeat until samples count is 2*K // while( samplescntall(double)(initbuf.ra0[ptidx]) ) { ptidx = i; } } for(i_=0; i_<=nvars-1;i_++) { ct[cidx+1,i_] = initbuf.rm0[ptidx,i_]; } } return; } // // Internal error // alglib.ap.assert(false, "SelectInitialCenters: internal error"); } /************************************************************************* This function "fixes" centers, i.e. replaces ones which have no neighbor points by new centers which have at least one neighbor. If it is impossible to fix centers (not enough distinct points in the dataset), this function returns False. INPUT PARAMETERS: XY - dataset, array [0..NPoints-1,0..NVars-1]. NPoints - points count, >=1 NVars - number of variables, NVars>=1 CT - centers K - number of centers, K>=1 InitBuf - internal buffer, possibly unitialized instance of APBuffers. It is recommended to use this instance only with SelectInitialCenters() and FixCenters() functions, because these functions may allocate really large storage. UpdatePool - shared pool seeded with instance of APBuffers structure (seed instance can be unitialized). Used internally with KMeansUpdateDistances() function. It is recommended to use this pool ONLY with KMeansUpdateDistances() function. OUTPUT PARAMETERS: CT - set of K centers, one per row RESULT: True on success, False on failure (impossible to create K independent clusters) -- ALGLIB -- Copyright 21.01.2015 by Bochkanov Sergey *************************************************************************/ private static bool fixcenters(double[,] xy, int npoints, int nvars, double[,] ct, int k, apserv.apbuffers initbuf, alglib.smp.shared_pool updatepool, alglib.xparams _params) { bool result = new bool(); int fixiteration = 0; int centertofix = 0; int i = 0; int j = 0; int pdistant = 0; double ddistant = 0; double v = 0; int i_ = 0; alglib.ap.assert(npoints>=1, "FixCenters: internal error"); alglib.ap.assert(nvars>=1, "FixCenters: internal error"); alglib.ap.assert(k>=1, "FixCenters: internal error"); // // Calculate distances from points to best centers (RA0) // and best center indexes (IA0) // apserv.ivectorsetlengthatleast(ref initbuf.ia0, npoints, _params); apserv.rvectorsetlengthatleast(ref initbuf.ra0, npoints, _params); kmeansupdatedistances(xy, 0, npoints, nvars, ct, 0, k, initbuf.ia0, initbuf.ra0, updatepool, _params); // // Repeat loop: // * find first center which has no corresponding point // * set it to the most distant (from the rest of the centerset) point // * recalculate distances, update IA0/RA0 // * repeat // // Loop is repeated for at most 2*K iterations. It is stopped once we have // no "empty" clusters. // apserv.bvectorsetlengthatleast(ref initbuf.ba0, k, _params); for(fixiteration=0; fixiteration<=2*k; fixiteration++) { // // Select center to fix (one which is not mentioned in IA0), // terminate if there is no such center. // BA0[] stores True for centers which have at least one point. // for(i=0; i<=k-1; i++) { initbuf.ba0[i] = false; } for(i=0; i<=npoints-1; i++) { initbuf.ba0[initbuf.ia0[i]] = true; } centertofix = -1; for(i=0; i<=k-1; i++) { if( !initbuf.ba0[i] ) { centertofix = i; break; } } if( centertofix<0 ) { result = true; return result; } // // Replace center to fix by the most distant point. // Update IA0/RA0 // pdistant = 0; ddistant = initbuf.ra0[pdistant]; for(i=0; i<=npoints-1; i++) { if( (double)(initbuf.ra0[i])>(double)(ddistant) ) { ddistant = initbuf.ra0[i]; pdistant = i; } } if( (double)(ddistant)==(double)(0.0) ) { break; } for(i_=0; i_<=nvars-1;i_++) { ct[centertofix,i_] = xy[pdistant,i_]; } for(i=0; i<=npoints-1; i++) { v = 0.0; for(j=0; j<=nvars-1; j++) { v = v+math.sqr(xy[i,j]-ct[centertofix,j]); } if( (double)(v)<(double)(initbuf.ra0[i]) ) { initbuf.ra0[i] = v; initbuf.ia0[i] = centertofix; } } } result = false; return result; } /************************************************************************* This function performs agglomerative hierarchical clustering using precomputed distance matrix. Internal function, should not be called directly. INPUT PARAMETERS: S - clusterizer state, initialized by ClusterizerCreate() D - distance matrix, array[S.NFeatures,S.NFeatures] Contents of the matrix is destroyed during algorithm operation. OUTPUT PARAMETERS: Rep - clustering results; see description of AHCReport structure for more information. -- ALGLIB -- Copyright 10.07.2012 by Bochkanov Sergey *************************************************************************/ private static void clusterizerrunahcinternal(clusterizerstate s, ref double[,] d, ahcreport rep, alglib.xparams _params) { int i = 0; int j = 0; int k = 0; double v = 0; int mergeidx = 0; int c0 = 0; int c1 = 0; int s0 = 0; int s1 = 0; int ar = 0; int br = 0; int npoints = 0; int[] cidx = new int[0]; int[] csizes = new int[0]; int[] nnidx = new int[0]; int[,] cinfo = new int[0,0]; int n0 = 0; int n1 = 0; int ni = 0; double d01 = 0; npoints = s.npoints; // // Fill Rep.NPoints, quick exit when NPoints<=1 // rep.npoints = npoints; if( npoints==0 ) { rep.p = new int[0]; rep.z = new int[0, 0]; rep.pz = new int[0, 0]; rep.pm = new int[0, 0]; rep.mergedist = new double[0]; rep.terminationtype = 1; return; } if( npoints==1 ) { rep.p = new int[1]; rep.z = new int[0, 0]; rep.pz = new int[0, 0]; rep.pm = new int[0, 0]; rep.mergedist = new double[0]; rep.p[0] = 0; rep.terminationtype = 1; return; } rep.z = new int[npoints-1, 2]; rep.mergedist = new double[npoints-1]; rep.terminationtype = 1; // // Build list of nearest neighbors // nnidx = new int[npoints]; for(i=0; i<=npoints-1; i++) { // // Calculate index of the nearest neighbor // k = -1; v = math.maxrealnumber; for(j=0; j<=npoints-1; j++) { if( j!=i && (double)(d[i,j])<(double)(v) ) { k = j; v = d[i,j]; } } alglib.ap.assert((double)(v)<(double)(math.maxrealnumber), "ClusterizerRunAHC: internal error"); nnidx[i] = k; } // // For AHCAlgo=4 (Ward's method) replace distances by their squares times 0.5 // if( s.ahcalgo==4 ) { for(i=0; i<=npoints-1; i++) { for(j=0; j<=npoints-1; j++) { d[i,j] = 0.5*d[i,j]*d[i,j]; } } } // // Distance matrix is built, perform merges. // // NOTE 1: CIdx is array[NPoints] which maps rows/columns of the // distance matrix D to indexes of clusters. Values of CIdx // from [0,NPoints) denote single-point clusters, and values // from [NPoints,2*NPoints-1) denote ones obtained by merging // smaller clusters. Negative calues correspond to absent clusters. // // Initially it contains [0...NPoints-1], after each merge // one element of CIdx (one with index C0) is replaced by // NPoints+MergeIdx, and another one with index C1 is // rewritten by -1. // // NOTE 2: CSizes is array[NPoints] which stores sizes of clusters. // // cidx = new int[npoints]; csizes = new int[npoints]; for(i=0; i<=npoints-1; i++) { cidx[i] = i; csizes[i] = 1; } for(mergeidx=0; mergeidx<=npoints-2; mergeidx++) { // // Select pair of clusters (C0,C1) with CIdx[C0]=0 ) { if( (double)(d[i,nnidx[i]])<(double)(d01) ) { c0 = i; c1 = nnidx[i]; d01 = d[i,nnidx[i]]; } } } alglib.ap.assert((double)(d01)<(double)(math.maxrealnumber), "ClusterizerRunAHC: internal error"); if( cidx[c0]>cidx[c1] ) { i = c1; c1 = c0; c0 = i; } // // Fill one row of Rep.Z and one element of Rep.MergeDist // rep.z[mergeidx,0] = cidx[c0]; rep.z[mergeidx,1] = cidx[c1]; rep.mergedist[mergeidx] = d01; // // Update distance matrix: // * row/column C0 are updated by distances to the new cluster // * row/column C1 are considered empty (we can fill them by zeros, // but do not want to spend time - we just ignore them) // // NOTE: it is important to update distance matrix BEFORE CIdx/CSizes // are updated. // alglib.ap.assert((((s.ahcalgo==0 || s.ahcalgo==1) || s.ahcalgo==2) || s.ahcalgo==3) || s.ahcalgo==4, "ClusterizerRunAHC: internal error"); for(i=0; i<=npoints-1; i++) { if( i!=c0 && i!=c1 ) { n0 = csizes[c0]; n1 = csizes[c1]; ni = csizes[i]; if( s.ahcalgo==0 ) { d[i,c0] = Math.Max(d[i,c0], d[i,c1]); } if( s.ahcalgo==1 ) { d[i,c0] = Math.Min(d[i,c0], d[i,c1]); } if( s.ahcalgo==2 ) { d[i,c0] = (csizes[c0]*d[i,c0]+csizes[c1]*d[i,c1])/(csizes[c0]+csizes[c1]); } if( s.ahcalgo==3 ) { d[i,c0] = (d[i,c0]+d[i,c1])/2; } if( s.ahcalgo==4 ) { d[i,c0] = ((n0+ni)*d[i,c0]+(n1+ni)*d[i,c1]-ni*d01)/(n0+n1+ni); } d[c0,i] = d[i,c0]; } } // // Update CIdx and CSizes // cidx[c0] = npoints+mergeidx; cidx[c1] = -1; csizes[c0] = csizes[c0]+csizes[c1]; csizes[c1] = 0; // // Update nearest neighbors array: // * update nearest neighbors of everything except for C0/C1 // * update neighbors of C0/C1 // for(i=0; i<=npoints-1; i++) { if( (cidx[i]>=0 && i!=c0) && (nnidx[i]==c0 || nnidx[i]==c1) ) { // // I-th cluster which is distinct from C0/C1 has former C0/C1 cluster as its nearest // neighbor. We handle this issue depending on specific AHC algorithm being used. // if( s.ahcalgo==1 ) { // // Single linkage. Merging of two clusters together // does NOT change distances between new cluster and // other clusters. // // The only thing we have to do is to update nearest neighbor index // nnidx[i] = c0; } else { // // Something other than single linkage. We have to re-examine // all the row to find nearest neighbor. // k = -1; v = math.maxrealnumber; for(j=0; j<=npoints-1; j++) { if( (cidx[j]>=0 && j!=i) && (double)(d[i,j])<(double)(v) ) { k = j; v = d[i,j]; } } alglib.ap.assert((double)(v)<(double)(math.maxrealnumber) || mergeidx==npoints-2, "ClusterizerRunAHC: internal error"); nnidx[i] = k; } } } k = -1; v = math.maxrealnumber; for(j=0; j<=npoints-1; j++) { if( (cidx[j]>=0 && j!=c0) && (double)(d[c0,j])<(double)(v) ) { k = j; v = d[c0,j]; } } alglib.ap.assert((double)(v)<(double)(math.maxrealnumber) || mergeidx==npoints-2, "ClusterizerRunAHC: internal error"); nnidx[c0] = k; } // // Calculate Rep.P and Rep.PM. // // In order to do that, we fill CInfo matrix - (2*NPoints-1)*3 matrix, // with I-th row containing: // * CInfo[I,0] - size of I-th cluster // * CInfo[I,1] - beginning of I-th cluster // * CInfo[I,2] - end of I-th cluster // * CInfo[I,3] - height of I-th cluster // // We perform it as follows: // * first NPoints clusters have unit size (CInfo[I,0]=1) and zero // height (CInfo[I,3]=0) // * we replay NPoints-1 merges from first to last and fill sizes of // corresponding clusters (new size is a sum of sizes of clusters // being merged) and height (new height is max(heights)+1). // * now we ready to determine locations of clusters. Last cluster // spans entire dataset, we know it. We replay merges from last to // first, during each merge we already know location of the merge // result, and we can position first cluster to the left part of // the result, and second cluster to the right part. // rep.p = new int[npoints]; rep.pm = new int[npoints-1, 6]; cinfo = new int[2*npoints-1, 4]; for(i=0; i<=npoints-1; i++) { cinfo[i,0] = 1; cinfo[i,3] = 0; } for(i=0; i<=npoints-2; i++) { cinfo[npoints+i,0] = cinfo[rep.z[i,0],0]+cinfo[rep.z[i,1],0]; cinfo[npoints+i,3] = Math.Max(cinfo[rep.z[i,0],3], cinfo[rep.z[i,1],3])+1; } cinfo[2*npoints-2,1] = 0; cinfo[2*npoints-2,2] = npoints-1; for(i=npoints-2; i>=0; i--) { // // We merge C0 which spans [A0,B0] and C1 (spans [A1,B1]), // with unknown A0, B0, A1, B1. However, we know that result // is CR, which spans [AR,BR] with known AR/BR, and we know // sizes of C0, C1, CR (denotes as S0, S1, SR). // c0 = rep.z[i,0]; c1 = rep.z[i,1]; s0 = cinfo[c0,0]; s1 = cinfo[c1,0]; ar = cinfo[npoints+i,1]; br = cinfo[npoints+i,2]; cinfo[c0,1] = ar; cinfo[c0,2] = ar+s0-1; cinfo[c1,1] = br-(s1-1); cinfo[c1,2] = br; rep.pm[i,0] = cinfo[c0,1]; rep.pm[i,1] = cinfo[c0,2]; rep.pm[i,2] = cinfo[c1,1]; rep.pm[i,3] = cinfo[c1,2]; rep.pm[i,4] = cinfo[c0,3]; rep.pm[i,5] = cinfo[c1,3]; } for(i=0; i<=npoints-1; i++) { alglib.ap.assert(cinfo[i,1]==cinfo[i,2]); rep.p[i] = cinfo[i,1]; } // // Calculate Rep.PZ // rep.pz = new int[npoints-1, 2]; for(i=0; i<=npoints-2; i++) { rep.pz[i,0] = rep.z[i,0]; rep.pz[i,1] = rep.z[i,1]; if( rep.pz[i,0]=1 DistType- distance function: * 0 Chebyshev distance (L-inf norm) * 1 city block distance (L1 norm) D - preallocated output matrix I0,I1 - half interval of rows to calculate: [I0,I1) is processed J0,J1 - half interval of cols to calculate: [J0,J1) is processed OUTPUT PARAMETERS: D - array[NPoints,NPoints], distance matrix upper triangle and main diagonal are initialized with data. NOTE: intersection of [I0,I1) and [J0,J1) may completely lie in upper triangle, only partially intersect with it, or have zero intersection. In any case, only intersection of submatrix given by [I0,I1)*[J0,J1) with upper triangle of the matrix is evaluated. Say, for 4x4 distance matrix A: * [0,2)*[0,2) will result in evaluation of A00, A01, A11 * [2,4)*[2,4) will result in evaluation of A22, A23, A32, A33 * [2,4)*[0,2) will result in evaluation of empty set of elements -- ALGLIB -- Copyright 07.04.2013 by Bochkanov Sergey *************************************************************************/ private static void evaluatedistancematrixrec(double[,] xy, int nfeatures, int disttype, double[,] d, int i0, int i1, int j0, int j1, alglib.xparams _params) { double rcomplexity = 0; int len0 = 0; int len1 = 0; int i = 0; int j = 0; int k = 0; double v = 0; double vv = 0; alglib.ap.assert(disttype==0 || disttype==1, "EvaluateDistanceMatrixRec: incorrect DistType"); // // Normalize J0/J1: // * J0:=max(J0,I0) - we ignore lower triangle // * J1:=max(J1,J0) - normalize J1 // j0 = Math.Max(j0, i0); j1 = Math.Max(j1, j0); if( j1<=j0 || i1<=i0 ) { return; } rcomplexity = complexitymultiplier*apserv.rmul3(i1-i0, j1-j0, nfeatures, _params); if( (i1-i0>2 || j1-j0>2) && (double)(rcomplexity)>=(double)(apserv.smpactivationlevel(_params)) ) { if( _trypexec_evaluatedistancematrixrec(xy,nfeatures,disttype,d,i0,i1,j0,j1, _params) ) { return; } } // // Try to process in parallel. Two condtions must hold in order to // activate parallel processing: // 1. I1-I0>2 or J1-J0>2 // 2. (I1-I0)*(J1-J0)*NFeatures>=ParallelComplexity // // NOTE: all quantities are converted to reals in order to avoid // integer overflow during multiplication // // NOTE: strict inequality in (1) is necessary to reduce task to 2x2 // basecases. In future versions we will be able to handle such // basecases more efficiently than 1x1 cases. // if( (double)(rcomplexity)>=(double)(apserv.spawnlevel(_params)) && (i1-i0>2 || j1-j0>2) ) { // // Recursive division along largest of dimensions // if( i1-i0>j1-j0 ) { apserv.splitlengtheven(i1-i0, ref len0, ref len1, _params); evaluatedistancematrixrec(xy, nfeatures, disttype, d, i0, i0+len0, j0, j1, _params); evaluatedistancematrixrec(xy, nfeatures, disttype, d, i0+len0, i1, j0, j1, _params); } else { apserv.splitlengtheven(j1-j0, ref len0, ref len1, _params); evaluatedistancematrixrec(xy, nfeatures, disttype, d, i0, i1, j0, j0+len0, _params); evaluatedistancematrixrec(xy, nfeatures, disttype, d, i0, i1, j0+len0, j1, _params); } return; } // // Sequential processing // for(i=i0; i<=i1-1; i++) { for(j=j0; j<=j1-1; j++) { if( j>=i ) { v = 0.0; if( disttype==0 ) { for(k=0; k<=nfeatures-1; k++) { vv = xy[i,k]-xy[j,k]; if( (double)(vv)<(double)(0) ) { vv = -vv; } if( (double)(vv)>(double)(v) ) { v = vv; } } } if( disttype==1 ) { for(k=0; k<=nfeatures-1; k++) { vv = xy[i,k]-xy[j,k]; if( (double)(vv)<(double)(0) ) { vv = -vv; } v = v+vv; } } d[i,j] = v; } } } } /************************************************************************* Serial stub for GPL edition. *************************************************************************/ public static bool _trypexec_evaluatedistancematrixrec(double[,] xy, int nfeatures, int disttype, double[,] d, int i0, int i1, int j0, int j1, alglib.xparams _params) { return false; } } public class dforest { /************************************************************************* A random forest (decision forest) builder object. Used to store dataset and specify decision forest training algorithm settings. *************************************************************************/ public class decisionforestbuilder : apobject { public int dstype; public int npoints; public int nvars; public int nclasses; public double[] dsdata; public double[] dsrval; public int[] dsival; public int rdfalgo; public double rdfratio; public double rdfvars; public int rdfglobalseed; public int rdfsplitstrength; public int rdfimportance; public double[] dsmin; public double[] dsmax; public bool[] dsbinary; public double dsravg; public int[] dsctotals; public int rdfprogress; public int rdftotal; public alglib.smp.shared_pool workpool; public alglib.smp.shared_pool votepool; public alglib.smp.shared_pool treepool; public alglib.smp.shared_pool treefactory; public bool neediobmatrix; public bool[,] iobmatrix; public int[] varimpshuffle2; public decisionforestbuilder() { init(); } public override void init() { dsdata = new double[0]; dsrval = new double[0]; dsival = new int[0]; dsmin = new double[0]; dsmax = new double[0]; dsbinary = new bool[0]; dsctotals = new int[0]; workpool = new alglib.smp.shared_pool(); votepool = new alglib.smp.shared_pool(); treepool = new alglib.smp.shared_pool(); treefactory = new alglib.smp.shared_pool(); iobmatrix = new bool[0,0]; varimpshuffle2 = new int[0]; } public override alglib.apobject make_copy() { decisionforestbuilder _result = new decisionforestbuilder(); _result.dstype = dstype; _result.npoints = npoints; _result.nvars = nvars; _result.nclasses = nclasses; _result.dsdata = (double[])dsdata.Clone(); _result.dsrval = (double[])dsrval.Clone(); _result.dsival = (int[])dsival.Clone(); _result.rdfalgo = rdfalgo; _result.rdfratio = rdfratio; _result.rdfvars = rdfvars; _result.rdfglobalseed = rdfglobalseed; _result.rdfsplitstrength = rdfsplitstrength; _result.rdfimportance = rdfimportance; _result.dsmin = (double[])dsmin.Clone(); _result.dsmax = (double[])dsmax.Clone(); _result.dsbinary = (bool[])dsbinary.Clone(); _result.dsravg = dsravg; _result.dsctotals = (int[])dsctotals.Clone(); _result.rdfprogress = rdfprogress; _result.rdftotal = rdftotal; _result.workpool = (alglib.smp.shared_pool)workpool.make_copy(); _result.votepool = (alglib.smp.shared_pool)votepool.make_copy(); _result.treepool = (alglib.smp.shared_pool)treepool.make_copy(); _result.treefactory = (alglib.smp.shared_pool)treefactory.make_copy(); _result.neediobmatrix = neediobmatrix; _result.iobmatrix = (bool[,])iobmatrix.Clone(); _result.varimpshuffle2 = (int[])varimpshuffle2.Clone(); return _result; } }; public class dfworkbuf : apobject { public int[] classpriors; public int[] varpool; public int varpoolsize; public int[] trnset; public int trnsize; public double[] trnlabelsr; public int[] trnlabelsi; public int[] oobset; public int oobsize; public double[] ooblabelsr; public int[] ooblabelsi; public double[] treebuf; public double[] curvals; public double[] bestvals; public int[] tmp0i; public int[] tmp1i; public double[] tmp0r; public double[] tmp1r; public double[] tmp2r; public double[] tmp3r; public int[] tmpnrms2; public int[] classtotals0; public int[] classtotals1; public int[] classtotals01; public dfworkbuf() { init(); } public override void init() { classpriors = new int[0]; varpool = new int[0]; trnset = new int[0]; trnlabelsr = new double[0]; trnlabelsi = new int[0]; oobset = new int[0]; ooblabelsr = new double[0]; ooblabelsi = new int[0]; treebuf = new double[0]; curvals = new double[0]; bestvals = new double[0]; tmp0i = new int[0]; tmp1i = new int[0]; tmp0r = new double[0]; tmp1r = new double[0]; tmp2r = new double[0]; tmp3r = new double[0]; tmpnrms2 = new int[0]; classtotals0 = new int[0]; classtotals1 = new int[0]; classtotals01 = new int[0]; } public override alglib.apobject make_copy() { dfworkbuf _result = new dfworkbuf(); _result.classpriors = (int[])classpriors.Clone(); _result.varpool = (int[])varpool.Clone(); _result.varpoolsize = varpoolsize; _result.trnset = (int[])trnset.Clone(); _result.trnsize = trnsize; _result.trnlabelsr = (double[])trnlabelsr.Clone(); _result.trnlabelsi = (int[])trnlabelsi.Clone(); _result.oobset = (int[])oobset.Clone(); _result.oobsize = oobsize; _result.ooblabelsr = (double[])ooblabelsr.Clone(); _result.ooblabelsi = (int[])ooblabelsi.Clone(); _result.treebuf = (double[])treebuf.Clone(); _result.curvals = (double[])curvals.Clone(); _result.bestvals = (double[])bestvals.Clone(); _result.tmp0i = (int[])tmp0i.Clone(); _result.tmp1i = (int[])tmp1i.Clone(); _result.tmp0r = (double[])tmp0r.Clone(); _result.tmp1r = (double[])tmp1r.Clone(); _result.tmp2r = (double[])tmp2r.Clone(); _result.tmp3r = (double[])tmp3r.Clone(); _result.tmpnrms2 = (int[])tmpnrms2.Clone(); _result.classtotals0 = (int[])classtotals0.Clone(); _result.classtotals1 = (int[])classtotals1.Clone(); _result.classtotals01 = (int[])classtotals01.Clone(); return _result; } }; public class dfvotebuf : apobject { public double[] trntotals; public double[] oobtotals; public int[] trncounts; public int[] oobcounts; public double[] giniimportances; public dfvotebuf() { init(); } public override void init() { trntotals = new double[0]; oobtotals = new double[0]; trncounts = new int[0]; oobcounts = new int[0]; giniimportances = new double[0]; } public override alglib.apobject make_copy() { dfvotebuf _result = new dfvotebuf(); _result.trntotals = (double[])trntotals.Clone(); _result.oobtotals = (double[])oobtotals.Clone(); _result.trncounts = (int[])trncounts.Clone(); _result.oobcounts = (int[])oobcounts.Clone(); _result.giniimportances = (double[])giniimportances.Clone(); return _result; } }; /************************************************************************* Permutation importance buffer object, stores permutation-related losses for some subset of the dataset + some temporaries Losses - array[NVars+2], stores sum of squared residuals for each permutation type: * Losses[0..NVars-1] stores losses for permutation in J-th variable * Losses[NVars] stores loss for all variables being randomly perturbed * Losses[NVars+1] stores loss for unperturbed dataset *************************************************************************/ public class dfpermimpbuf : apobject { public double[] losses; public double[] xraw; public double[] xdist; public double[] xcur; public double[] y; public double[] yv; public double[] targety; public int[] startnodes; public dfpermimpbuf() { init(); } public override void init() { losses = new double[0]; xraw = new double[0]; xdist = new double[0]; xcur = new double[0]; y = new double[0]; yv = new double[0]; targety = new double[0]; startnodes = new int[0]; } public override alglib.apobject make_copy() { dfpermimpbuf _result = new dfpermimpbuf(); _result.losses = (double[])losses.Clone(); _result.xraw = (double[])xraw.Clone(); _result.xdist = (double[])xdist.Clone(); _result.xcur = (double[])xcur.Clone(); _result.y = (double[])y.Clone(); _result.yv = (double[])yv.Clone(); _result.targety = (double[])targety.Clone(); _result.startnodes = (int[])startnodes.Clone(); return _result; } }; public class dftreebuf : apobject { public double[] treebuf; public int treeidx; public dftreebuf() { init(); } public override void init() { treebuf = new double[0]; } public override alglib.apobject make_copy() { dftreebuf _result = new dftreebuf(); _result.treebuf = (double[])treebuf.Clone(); _result.treeidx = treeidx; return _result; } }; /************************************************************************* Buffer object which is used to perform various requests (usually model inference) in the multithreaded mode (multiple threads working with same DF object). This object should be created with DFCreateBuffer(). *************************************************************************/ public class decisionforestbuffer : apobject { public double[] x; public double[] y; public decisionforestbuffer() { init(); } public override void init() { x = new double[0]; y = new double[0]; } public override alglib.apobject make_copy() { decisionforestbuffer _result = new decisionforestbuffer(); _result.x = (double[])x.Clone(); _result.y = (double[])y.Clone(); return _result; } }; /************************************************************************* Decision forest (random forest) model. *************************************************************************/ public class decisionforest : apobject { public int forestformat; public bool usemantissa8; public int nvars; public int nclasses; public int ntrees; public int bufsize; public double[] trees; public decisionforestbuffer buffer; public byte[] trees8; public decisionforest() { init(); } public override void init() { trees = new double[0]; buffer = new decisionforestbuffer(); trees8 = new byte[0]; } public override alglib.apobject make_copy() { decisionforest _result = new decisionforest(); _result.forestformat = forestformat; _result.usemantissa8 = usemantissa8; _result.nvars = nvars; _result.nclasses = nclasses; _result.ntrees = ntrees; _result.bufsize = bufsize; _result.trees = (double[])trees.Clone(); _result.buffer = (decisionforestbuffer)buffer.make_copy(); _result.trees8 = (byte[])trees8.Clone(); return _result; } }; /************************************************************************* Decision forest training report. === training/oob errors ================================================== Following fields store training set errors: * relclserror - fraction of misclassified cases, [0,1] * avgce - average cross-entropy in bits per symbol * rmserror - root-mean-square error * avgerror - average error * avgrelerror - average relative error Out-of-bag estimates are stored in fields with same names, but "oob" prefix. For classification problems: * RMS, AVG and AVGREL errors are calculated for posterior probabilities For regression problems: * RELCLS and AVGCE errors are zero === variable importance ================================================== Following fields are used to store variable importance information: * topvars - variables ordered from the most important to less important ones (according to current choice of importance raiting). For example, topvars[0] contains index of the most important variable, and topvars[0:2] are indexes of 3 most important ones and so on. * varimportances - array[nvars], ratings (the larger, the more important the variable is, always in [0,1] range). By default, filled by zeros (no importance ratings are provided unless you explicitly request them). Zero rating means that variable is not important, however you will rarely encounter such a thing, in many cases unimportant variables produce nearly-zero (but nonzero) ratings. Variable importance report must be EXPLICITLY requested by calling: * dfbuildersetimportancegini() function, if you need out-of-bag Gini-based importance rating also known as MDI (fast to calculate, resistant to overfitting issues, but has some bias towards continuous and high-cardinality categorical variables) * dfbuildersetimportancetrngini() function, if you need training set Gini- -based importance rating (what other packages typically report). * dfbuildersetimportancepermutation() function, if you need permutation- based importance rating also known as MDA (slower to calculate, but less biased) * dfbuildersetimportancenone() function, if you do not need importance ratings - ratings will be zero, topvars[] will be [0,1,2,...] Different importance ratings (Gini or permutation) produce non-comparable values. Although in all cases rating values lie in [0,1] range, there are exist differences: * informally speaking, Gini importance rating tends to divide "unit amount of importance" between several important variables, i.e. it produces estimates which roughly sum to 1.0 (or less than 1.0, if your task can not be solved exactly). If all variables are equally important, they will have same rating, roughly 1/NVars, even if every variable is critically important. * from the other side, permutation importance tells us what percentage of the model predictive power will be ruined by permuting this specific variable. It does not produce estimates which sum to one. Critically important variable will have rating close to 1.0, and you may have multiple variables with such a rating. More information on variable importance ratings can be found in comments on the dfbuildersetimportancegini() and dfbuildersetimportancepermutation() functions. *************************************************************************/ public class dfreport : apobject { public double relclserror; public double avgce; public double rmserror; public double avgerror; public double avgrelerror; public double oobrelclserror; public double oobavgce; public double oobrmserror; public double oobavgerror; public double oobavgrelerror; public int[] topvars; public double[] varimportances; public dfreport() { init(); } public override void init() { topvars = new int[0]; varimportances = new double[0]; } public override alglib.apobject make_copy() { dfreport _result = new dfreport(); _result.relclserror = relclserror; _result.avgce = avgce; _result.rmserror = rmserror; _result.avgerror = avgerror; _result.avgrelerror = avgrelerror; _result.oobrelclserror = oobrelclserror; _result.oobavgce = oobavgce; _result.oobrmserror = oobrmserror; _result.oobavgerror = oobavgerror; _result.oobavgrelerror = oobavgrelerror; _result.topvars = (int[])topvars.Clone(); _result.varimportances = (double[])varimportances.Clone(); return _result; } }; public class dfinternalbuffers : apobject { public double[] treebuf; public int[] idxbuf; public double[] tmpbufr; public double[] tmpbufr2; public int[] tmpbufi; public int[] classibuf; public double[] sortrbuf; public double[] sortrbuf2; public int[] sortibuf; public int[] varpool; public bool[] evsbin; public double[] evssplits; public dfinternalbuffers() { init(); } public override void init() { treebuf = new double[0]; idxbuf = new int[0]; tmpbufr = new double[0]; tmpbufr2 = new double[0]; tmpbufi = new int[0]; classibuf = new int[0]; sortrbuf = new double[0]; sortrbuf2 = new double[0]; sortibuf = new int[0]; varpool = new int[0]; evsbin = new bool[0]; evssplits = new double[0]; } public override alglib.apobject make_copy() { dfinternalbuffers _result = new dfinternalbuffers(); _result.treebuf = (double[])treebuf.Clone(); _result.idxbuf = (int[])idxbuf.Clone(); _result.tmpbufr = (double[])tmpbufr.Clone(); _result.tmpbufr2 = (double[])tmpbufr2.Clone(); _result.tmpbufi = (int[])tmpbufi.Clone(); _result.classibuf = (int[])classibuf.Clone(); _result.sortrbuf = (double[])sortrbuf.Clone(); _result.sortrbuf2 = (double[])sortrbuf2.Clone(); _result.sortibuf = (int[])sortibuf.Clone(); _result.varpool = (int[])varpool.Clone(); _result.evsbin = (bool[])evsbin.Clone(); _result.evssplits = (double[])evssplits.Clone(); return _result; } }; public const int innernodewidth = 3; public const int leafnodewidth = 2; public const int dfusestrongsplits = 1; public const int dfuseevs = 2; public const int dfuncompressedv0 = 0; public const int dfcompressedv0 = 1; public const int needtrngini = 1; public const int needoobgini = 2; public const int needpermutation = 3; public const int permutationimportancebatchsize = 512; /************************************************************************* This function creates buffer structure which can be used to perform parallel inference requests. DF subpackage provides two sets of computing functions - ones which use internal buffer of DF model (these functions are single-threaded because they use same buffer, which can not shared between threads), and ones which use external buffer. This function is used to initialize external buffer. INPUT PARAMETERS Model - DF model which is associated with newly created buffer OUTPUT PARAMETERS Buf - external buffer. IMPORTANT: buffer object should be used only with model which was used to initialize buffer. Any attempt to use buffer with different object is dangerous - you may get integrity check failure (exception) because sizes of internal arrays do not fit to dimensions of the model structure. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void dfcreatebuffer(decisionforest model, decisionforestbuffer buf, alglib.xparams _params) { buf.x = new double[model.nvars]; buf.y = new double[model.nclasses]; } /************************************************************************* This subroutine creates DecisionForestBuilder object which is used to train decision forests. By default, new builder stores empty dataset and some reasonable default settings. At the very least, you should specify dataset prior to building decision forest. You can also tweak settings of the forest construction algorithm (recommended, although default setting should work well). Following actions are mandatory: * calling dfbuildersetdataset() to specify dataset * calling dfbuilderbuildrandomforest() to build decision forest using current dataset and default settings Additionally, you may call: * dfbuildersetrndvars() or dfbuildersetrndvarsratio() to specify number of variables randomly chosen for each split * dfbuildersetsubsampleratio() to specify fraction of the dataset randomly subsampled to build each tree * dfbuildersetseed() to control random seed chosen for tree construction INPUT PARAMETERS: none OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildercreate(decisionforestbuilder s, alglib.xparams _params) { // // Empty dataset // s.dstype = -1; s.npoints = 0; s.nvars = 0; s.nclasses = 1; // // Default training settings // s.rdfalgo = 0; s.rdfratio = 0.5; s.rdfvars = 0.0; s.rdfglobalseed = 0; s.rdfsplitstrength = 2; s.rdfimportance = 0; // // Other fields // s.rdfprogress = 0; s.rdftotal = 1; } /************************************************************************* This subroutine adds dense dataset to the internal storage of the builder object. Specifying your dataset in the dense format means that the dense version of the forest construction algorithm will be invoked. INPUT PARAMETERS: S - decision forest builder object XY - array[NPoints,NVars+1] (minimum size; actual size can be larger, only leading part is used anyway), dataset: * first NVars elements of each row store values of the independent variables * last column store class number (in 0...NClasses-1) or real value of the dependent variable NPoints - number of rows in the dataset, NPoints>=1 NVars - number of independent variables, NVars>=1 NClasses - indicates type of the problem being solved: * NClasses>=2 means that classification problem is solved (last column of the dataset stores class number) * NClasses=1 means that regression problem is solved (last column of the dataset stores variable value) OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetdataset(decisionforestbuilder s, double[,] xy, int npoints, int nvars, int nclasses, alglib.xparams _params) { int i = 0; int j = 0; // // Check parameters // alglib.ap.assert(npoints>=1, "dfbuildersetdataset: npoints<1"); alglib.ap.assert(nvars>=1, "dfbuildersetdataset: nvars<1"); alglib.ap.assert(nclasses>=1, "dfbuildersetdataset: nclasses<1"); alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "dfbuildersetdataset: rows(xy)=nvars+1, "dfbuildersetdataset: cols(xy)1 ) { for(i=0; i<=npoints-1; i++) { j = (int)Math.Round(xy[i,nvars]); alglib.ap.assert(j>=0 && j1 ) { apserv.ivectorsetlengthatleast(ref s.dsival, npoints, _params); for(i=0; i<=npoints-1; i++) { s.dsival[i] = (int)Math.Round(xy[i,nvars]); } } else { apserv.rvectorsetlengthatleast(ref s.dsrval, npoints, _params); for(i=0; i<=npoints-1; i++) { s.dsrval[i] = xy[i,nvars]; } } } /************************************************************************* This function sets number of variables (in [1,NVars] range) used by decision forest construction algorithm. The default option is to use roughly sqrt(NVars) variables. INPUT PARAMETERS: S - decision forest builder object RndVars - number of randomly selected variables; values outside of [1,NVars] range are silently clipped. OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrndvars(decisionforestbuilder s, int rndvars, alglib.xparams _params) { s.rdfvars = Math.Max(rndvars, 1); } /************************************************************************* This function sets number of variables used by decision forest construction algorithm as a fraction of total variable count (0,1) range. The default option is to use roughly sqrt(NVars) variables. INPUT PARAMETERS: S - decision forest builder object F - round(NVars*F) variables are selected OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrndvarsratio(decisionforestbuilder s, double f, alglib.xparams _params) { alglib.ap.assert(math.isfinite(f), "dfbuildersetrndvarsratio: F is INF or NAN"); s.rdfvars = -Math.Max(f, math.machineepsilon); } /************************************************************************* This function tells decision forest builder to automatically choose number of variables used by decision forest construction algorithm. Roughly sqrt(NVars) variables will be used. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrndvarsauto(decisionforestbuilder s, alglib.xparams _params) { s.rdfvars = 0; } /************************************************************************* This function sets size of dataset subsample generated the decision forest construction algorithm. Size is specified as a fraction of total dataset size. The default option is to use 50% of the dataset for training, 50% for the OOB estimates. You can decrease fraction F down to 10%, 1% or even below in order to reduce overfitting. INPUT PARAMETERS: S - decision forest builder object F - fraction of the dataset to use, in (0,1] range. Values outside of this range will be silently clipped. At least one element is always selected for the training set. OUTPUT PARAMETERS: S - decision forest builder -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetsubsampleratio(decisionforestbuilder s, double f, alglib.xparams _params) { alglib.ap.assert(math.isfinite(f), "dfbuildersetrndvarsfraction: F is INF or NAN"); s.rdfratio = Math.Max(f, math.machineepsilon); } /************************************************************************* This function sets seed used by internal RNG for random subsampling and random selection of variable subsets. By default random seed is used, i.e. every time you build decision forest, we seed generator with new value obtained from system-wide RNG. Thus, decision forest builder returns non-deterministic results. You can change such behavior by specyfing fixed positive seed value. INPUT PARAMETERS: S - decision forest builder object SeedVal - seed value: * positive values are used for seeding RNG with fixed seed, i.e. subsequent runs on same data will return same decision forests * non-positive seed means that random seed is used for every run of builder, i.e. subsequent runs on same datasets will return slightly different decision forests OUTPUT PARAMETERS: S - decision forest builder, see -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetseed(decisionforestbuilder s, int seedval, alglib.xparams _params) { s.rdfglobalseed = seedval; } /************************************************************************* This function sets random decision forest construction algorithm. As for now, only one decision forest construction algorithm is supported - a dense "baseline" RDF algorithm. INPUT PARAMETERS: S - decision forest builder object AlgoType - algorithm type: * 0 = baseline dense RDF OUTPUT PARAMETERS: S - decision forest builder, see -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrdfalgo(decisionforestbuilder s, int algotype, alglib.xparams _params) { alglib.ap.assert(algotype==0, "dfbuildersetrdfalgo: unexpected algotype"); s.rdfalgo = algotype; } /************************************************************************* This function sets split selection algorithm used by decision forest classifier. You may choose several algorithms, with different speed and quality of the results. INPUT PARAMETERS: S - decision forest builder object SplitStrength- split type: * 0 = split at the random position, fastest one * 1 = split at the middle of the range * 2 = strong split at the best point of the range (default) OUTPUT PARAMETERS: S - decision forest builder, see -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetrdfsplitstrength(decisionforestbuilder s, int splitstrength, alglib.xparams _params) { alglib.ap.assert((splitstrength==0 || splitstrength==1) || splitstrength==2, "dfbuildersetrdfsplitstrength: unexpected split type"); s.rdfsplitstrength = splitstrength; } /************************************************************************* This function tells decision forest construction algorithm to use Gini impurity based variable importance estimation (also known as MDI). This version of importance estimation algorithm analyzes mean decrease in impurity (MDI) on training sample during splits. The result is divided by impurity at the root node in order to produce estimate in [0,1] range. Such estimates are fast to calculate and beautifully normalized (sum to one) but have following downsides: * They ALWAYS sum to 1.0, even if output is completely unpredictable. I.e. MDI allows to order variables by importance, but does not tell us about "absolute" importances of variables * there exist some bias towards continuous and high-cardinality categorical variables NOTE: informally speaking, MDA (permutation importance) rating answers the question "what part of the model predictive power is ruined by permuting k-th variable?" while MDI tells us "what part of the model predictive power was achieved due to usage of k-th variable". Thus, MDA rates each variable independently at "0 to 1" scale while MDI (and OOB-MDI too) tends to divide "unit amount of importance" between several important variables. If all variables are equally important, they will have same MDI/OOB-MDI rating, equal (for OOB-MDI: roughly equal) to 1/NVars. However, roughly same picture will be produced for the "all variables provide information no one is critical" situation and for the "all variables are critical, drop any one, everything is ruined" situation. Contrary to that, MDA will rate critical variable as ~1.0 important, and important but non-critical variable will have less than unit rating. NOTE: quite an often MDA and MDI return same results. It generally happens on problems with low test set error (a few percents at most) and large enough training set to avoid overfitting. The difference between MDA, MDI and OOB-MDI becomes important only on "hard" tasks with high test set error and/or small training set. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder object. Next call to the forest construction function will produce: * importance estimates in rep.varimportances field * variable ranks in rep.topvars field -- ALGLIB -- Copyright 29.07.2019 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetimportancetrngini(decisionforestbuilder s, alglib.xparams _params) { s.rdfimportance = needtrngini; } /************************************************************************* This function tells decision forest construction algorithm to use out-of-bag version of Gini variable importance estimation (also known as OOB-MDI). This version of importance estimation algorithm analyzes mean decrease in impurity (MDI) on out-of-bag sample during splits. The result is divided by impurity at the root node in order to produce estimate in [0,1] range. Such estimates are fast to calculate and resistant to overfitting issues (thanks to the out-of-bag estimates used). However, OOB Gini rating has following downsides: * there exist some bias towards continuous and high-cardinality categorical variables * Gini rating allows us to order variables by importance, but it is hard to define importance of the variable by itself. NOTE: informally speaking, MDA (permutation importance) rating answers the question "what part of the model predictive power is ruined by permuting k-th variable?" while MDI tells us "what part of the model predictive power was achieved due to usage of k-th variable". Thus, MDA rates each variable independently at "0 to 1" scale while MDI (and OOB-MDI too) tends to divide "unit amount of importance" between several important variables. If all variables are equally important, they will have same MDI/OOB-MDI rating, equal (for OOB-MDI: roughly equal) to 1/NVars. However, roughly same picture will be produced for the "all variables provide information no one is critical" situation and for the "all variables are critical, drop any one, everything is ruined" situation. Contrary to that, MDA will rate critical variable as ~1.0 important, and important but non-critical variable will have less than unit rating. NOTE: quite an often MDA and MDI return same results. It generally happens on problems with low test set error (a few percents at most) and large enough training set to avoid overfitting. The difference between MDA, MDI and OOB-MDI becomes important only on "hard" tasks with high test set error and/or small training set. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder object. Next call to the forest construction function will produce: * importance estimates in rep.varimportances field * variable ranks in rep.topvars field -- ALGLIB -- Copyright 29.07.2019 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetimportanceoobgini(decisionforestbuilder s, alglib.xparams _params) { s.rdfimportance = needoobgini; } /************************************************************************* This function tells decision forest construction algorithm to use permutation variable importance estimator (also known as MDA). This version of importance estimation algorithm analyzes mean increase in out-of-bag sum of squared residuals after random permutation of J-th variable. The result is divided by error computed with all variables being perturbed in order to produce R-squared-like estimate in [0,1] range. Such estimate is slower to calculate than Gini-based rating because it needs multiple inference runs for each of variables being studied. ALGLIB uses parallelized and highly optimized algorithm which analyzes path through the decision tree and allows to handle most perturbations in O(1) time; nevertheless, requesting MDA importances may increase forest construction time from 10% to 200% (or more, if you have thousands of variables). However, MDA rating has following benefits over Gini-based ones: * no bias towards specific variable types * ability to directly evaluate "absolute" importance of some variable at "0 to 1" scale (contrary to Gini-based rating, which returns comparative importances). NOTE: informally speaking, MDA (permutation importance) rating answers the question "what part of the model predictive power is ruined by permuting k-th variable?" while MDI tells us "what part of the model predictive power was achieved due to usage of k-th variable". Thus, MDA rates each variable independently at "0 to 1" scale while MDI (and OOB-MDI too) tends to divide "unit amount of importance" between several important variables. If all variables are equally important, they will have same MDI/OOB-MDI rating, equal (for OOB-MDI: roughly equal) to 1/NVars. However, roughly same picture will be produced for the "all variables provide information no one is critical" situation and for the "all variables are critical, drop any one, everything is ruined" situation. Contrary to that, MDA will rate critical variable as ~1.0 important, and important but non-critical variable will have less than unit rating. NOTE: quite an often MDA and MDI return same results. It generally happens on problems with low test set error (a few percents at most) and large enough training set to avoid overfitting. The difference between MDA, MDI and OOB-MDI becomes important only on "hard" tasks with high test set error and/or small training set. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder object. Next call to the forest construction function will produce: * importance estimates in rep.varimportances field * variable ranks in rep.topvars field -- ALGLIB -- Copyright 29.07.2019 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetimportancepermutation(decisionforestbuilder s, alglib.xparams _params) { s.rdfimportance = needpermutation; } /************************************************************************* This function tells decision forest construction algorithm to skip variable importance estimation. INPUT PARAMETERS: S - decision forest builder object OUTPUT PARAMETERS: S - decision forest builder object. Next call to the forest construction function will result in forest being built without variable importance estimation. -- ALGLIB -- Copyright 29.07.2019 by Bochkanov Sergey *************************************************************************/ public static void dfbuildersetimportancenone(decisionforestbuilder s, alglib.xparams _params) { s.rdfimportance = 0; } /************************************************************************* This function is an alias for dfbuilderpeekprogress(), left in ALGLIB for backward compatibility reasons. -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static double dfbuildergetprogress(decisionforestbuilder s, alglib.xparams _params) { double result = 0; result = dfbuilderpeekprogress(s, _params); return result; } /************************************************************************* This function is used to peek into decision forest construction process from some other thread and get current progress indicator. It returns value in [0,1]. INPUT PARAMETERS: S - decision forest builder object used to build forest in some other thread RESULT: progress value, in [0,1] -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static double dfbuilderpeekprogress(decisionforestbuilder s, alglib.xparams _params) { double result = 0; result = s.rdfprogress/Math.Max(s.rdftotal, 1); result = Math.Max(result, 0); result = Math.Min(result, 1); return result; } /************************************************************************* This subroutine builds decision forest according to current settings using dataset internally stored in the builder object. Dense algorithm is used. NOTE: this function uses dense algorithm for forest construction independently from the dataset format (dense or sparse). NOTE: forest built with this function is stored in-memory using 64-bit data structures for offsets/indexes/split values. It is possible to convert forest into more memory-efficient compressed binary representation. Depending on the problem properties, 3.7x-5.7x compression factors are possible. The downsides of compression are (a) slight reduction in the model accuracy and (b) ~1.5x reduction in the inference speed (due to increased complexity of the storage format). See comments on dfbinarycompression() for more info. Default settings are used by the algorithm; you can tweak them with the help of the following functions: * dfbuildersetrfactor() - to control a fraction of the dataset used for subsampling * dfbuildersetrandomvars() - to control number of variables randomly chosen for decision rule creation ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - decision forest builder object NTrees - NTrees>=1, number of trees to train OUTPUT PARAMETERS: DF - decision forest. You can compress this forest to more compact 16-bit representation with dfbinarycompression() Rep - report, see below for information on its fields. === report information produced by forest construction function ========== Decision forest training report includes following information: * training set errors * out-of-bag estimates of errors * variable importance ratings Following fields are used to store information: * training set errors are stored in rep.relclserror, rep.avgce, rep.rmserror, rep.avgerror and rep.avgrelerror * out-of-bag estimates of errors are stored in rep.oobrelclserror, rep.oobavgce, rep.oobrmserror, rep.oobavgerror and rep.oobavgrelerror Variable importance reports, if requested by dfbuildersetimportancegini(), dfbuildersetimportancetrngini() or dfbuildersetimportancepermutation() call, are stored in: * rep.varimportances field stores importance ratings * rep.topvars stores variable indexes ordered from the most important to less important ones You can find more information about report fields in: * comments on dfreport structure * comments on dfbuildersetimportancegini function * comments on dfbuildersetimportancetrngini function * comments on dfbuildersetimportancepermutation function -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ public static void dfbuilderbuildrandomforest(decisionforestbuilder s, int ntrees, decisionforest df, dfreport rep, alglib.xparams _params) { int i = 0; int j = 0; int nvars = 0; int nclasses = 0; int npoints = 0; int trnsize = 0; int maxtreesize = 0; int sessionseed = 0; dfworkbuf workbufseed = new dfworkbuf(); dfvotebuf votebufseed = new dfvotebuf(); dftreebuf treebufseed = new dftreebuf(); alglib.ap.assert(ntrees>=1, "DFBuilderBuildRandomForest: ntrees<1"); cleanreport(s, rep, _params); npoints = s.npoints; nvars = s.nvars; nclasses = s.nclasses; // // Set up progress counter // s.rdfprogress = 0; s.rdftotal = ntrees*npoints; if( s.rdfimportance==needpermutation ) { s.rdftotal = s.rdftotal+ntrees*npoints; } // // Quick exit for empty dataset // if( s.dstype==-1 || npoints==0 ) { alglib.ap.assert(leafnodewidth==2, "DFBuilderBuildRandomForest: integrity check failed"); df.forestformat = dfuncompressedv0; df.nvars = s.nvars; df.nclasses = s.nclasses; df.ntrees = 1; df.bufsize = 1+leafnodewidth; df.trees = new double[1+leafnodewidth]; df.trees[0] = 1+leafnodewidth; df.trees[1] = -1; df.trees[2] = 0.0; dfcreatebuffer(df, df.buffer, _params); return; } alglib.ap.assert(npoints>0, "DFBuilderBuildRandomForest: integrity check failed"); // // Analyze dataset statistics, perform preprocessing // analyzeandpreprocessdataset(s, _params); // // Prepare "work", "vote" and "tree" pools and other settings // trnsize = (int)Math.Round(npoints*s.rdfratio); trnsize = Math.Max(trnsize, 1); trnsize = Math.Min(trnsize, npoints); maxtreesize = 1+innernodewidth*(trnsize-1)+leafnodewidth*trnsize; workbufseed.varpool = new int[nvars]; workbufseed.trnset = new int[trnsize]; workbufseed.oobset = new int[npoints-trnsize]; workbufseed.tmp0i = new int[npoints]; workbufseed.tmp1i = new int[npoints]; workbufseed.tmp0r = new double[npoints]; workbufseed.tmp1r = new double[npoints]; workbufseed.tmp2r = new double[npoints]; workbufseed.tmp3r = new double[npoints]; workbufseed.trnlabelsi = new int[npoints]; workbufseed.trnlabelsr = new double[npoints]; workbufseed.ooblabelsi = new int[npoints]; workbufseed.ooblabelsr = new double[npoints]; workbufseed.curvals = new double[npoints]; workbufseed.bestvals = new double[npoints]; workbufseed.classpriors = new int[nclasses]; workbufseed.classtotals0 = new int[nclasses]; workbufseed.classtotals1 = new int[nclasses]; workbufseed.classtotals01 = new int[2*nclasses]; workbufseed.treebuf = new double[maxtreesize]; workbufseed.trnsize = trnsize; workbufseed.oobsize = npoints-trnsize; votebufseed.trntotals = new double[npoints*nclasses]; votebufseed.oobtotals = new double[npoints*nclasses]; for(i=0; i<=npoints*nclasses-1; i++) { votebufseed.trntotals[i] = 0; votebufseed.oobtotals[i] = 0; } votebufseed.trncounts = new int[npoints]; votebufseed.oobcounts = new int[npoints]; for(i=0; i<=npoints-1; i++) { votebufseed.trncounts[i] = 0; votebufseed.oobcounts[i] = 0; } votebufseed.giniimportances = new double[nvars]; for(i=0; i<=nvars-1; i++) { votebufseed.giniimportances[i] = 0.0; } treebufseed.treeidx = -1; alglib.smp.ae_shared_pool_set_seed(s.workpool, workbufseed); alglib.smp.ae_shared_pool_set_seed(s.votepool, votebufseed); alglib.smp.ae_shared_pool_set_seed(s.treepool, treebufseed); alglib.smp.ae_shared_pool_set_seed(s.treefactory, treebufseed); // // Select session seed (individual trees are constructed using // combination of session and local seeds). // sessionseed = s.rdfglobalseed; if( s.rdfglobalseed<=0 ) { sessionseed = math.randominteger(30000); } // // Prepare In-and-Out-of-Bag matrix, if needed // s.neediobmatrix = s.rdfimportance==needpermutation; if( s.neediobmatrix ) { // // Prepare default state of In-and-Out-of-Bag matrix // apserv.bmatrixsetlengthatleast(ref s.iobmatrix, ntrees, npoints, _params); for(i=0; i<=ntrees-1; i++) { for(j=0; j<=npoints-1; j++) { s.iobmatrix[i,j] = false; } } } // // Build trees (in parallel, if possible) // buildrandomtree(s, 0, ntrees, _params); // // Merge trees and output result // mergetrees(s, df, _params); // // Process voting results and output training set and OOB errors. // Finalize tree construction. // processvotingresults(s, ntrees, votebufseed, rep, _params); dfcreatebuffer(df, df.buffer, _params); // // Perform variable importance estimation // estimatevariableimportance(s, sessionseed, df, ntrees, rep, _params); // // Update progress counter // s.rdfprogress = s.rdftotal; } /************************************************************************* This function performs binary compression of the decision forest. Original decision forest produced by the forest builder is stored using 64-bit representation for all numbers - offsets, variable indexes, split points. It is possible to significantly reduce model size by means of: * using compressed dynamic encoding for integers (offsets and variable indexes), which uses just 1 byte to store small ints (less than 128), just 2 bytes for larger values (less than 128^2) and so on * storing floating point numbers using 8-bit exponent and 16-bit mantissa As result, model needs significantly less memory (compression factor depends on variable and class counts). In particular: * NVars<128 and NClasses<128 result in 4.4x-5.7x model size reduction * NVars<16384 and NClasses<128 result in 3.7x-4.5x model size reduction Such storage format performs lossless compression of all integers, but compression of floating point values (split values) is lossy, with roughly 0.01% relative error introduced during rounding. Thus, we recommend you to re-evaluate model accuracy after compression. Another downside of compression is ~1.5x reduction in the inference speed due to necessity of dynamic decompression of the compressed model. INPUT PARAMETERS: DF - decision forest built by forest builder OUTPUT PARAMETERS: DF - replaced by compressed forest RESULT: compression factor (in-RAM size of the compressed model vs than of the uncompressed one), positive number larger than 1.0 -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ public static double dfbinarycompression(decisionforest df, alglib.xparams _params) { double result = 0; result = binarycompression(df, false, _params); return result; } /************************************************************************* This is a 8-bit version of dfbinarycompression. Not recommended for external use because it is too lossy. -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ public static double dfbinarycompression8(decisionforest df, alglib.xparams _params) { double result = 0; result = binarycompression(df, true, _params); return result; } // HEAL pfleck /************************************************************************* Procesing INPUT PARAMETERS: DF - decision forest model X - input vector, array[0..NVars-1]. OUTPUT PARAMETERS: ys - result. Regression estimates for each tree when solving regression task. -- HeuristicLab -- Copyright 18.07.2016 by HEAL, based on implementation of dfprocess by Bochkanov Sergey *************************************************************************/ public static void dfprocessraw(decisionforest df, double[] x, ref double[] ys, alglib.xparams _params) { int offs = 0; int i = 0; var y = new double[1]; if (df.nclasses != 1) return; // // Proceed // if (alglib.ap.len(ys)model.buffer.y[result] ) { result = i; } } return result; } /************************************************************************* Inference using decision forest Thread-safe procesing using external buffer for temporaries. This function is thread-safe (i.e . you can use same DF model from multiple threads) as long as you use different buffer objects for different threads. INPUT PARAMETERS: DF - decision forest model Buf - buffer object, must be allocated specifically for this model with dfcreatebuffer(). X - input vector, array[NVars] Y - possibly preallocated buffer, reallocated if too small OUTPUT PARAMETERS: Y - result. Regression estimate when solving regression task, vector of posterior probabilities for classification task. See also DFProcessI. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static void dftsprocess(decisionforest df, decisionforestbuffer buf, double[] x, ref double[] y, alglib.xparams _params) { // // Although docs warn you about thread-unsafety of the dfprocess() // function, it is de facto thread-safe. However, thread safety is // an accidental side-effect of the specific inference algorithm // being used. It may disappear in the future versions of the DF // models, so you should NOT rely on it. // dfprocess(df, x, ref y, _params); } /************************************************************************* Relative classification error on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: percent of incorrectly classified cases. Zero if model solves regression task. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfrelclserror(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; result = (double)dfclserror(df, xy, npoints, _params)/(double)npoints; return result; } /************************************************************************* Average cross-entropy (in bits per element) on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: CrossEntropy/(NPoints*LN(2)). Zero if model solves regression task. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfavgce(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; double[] x = new double[0]; double[] y = new double[0]; int i = 0; int j = 0; int k = 0; int tmpi = 0; int i_ = 0; x = new double[df.nvars-1+1]; y = new double[df.nclasses-1+1]; result = 0; for(i=0; i<=npoints-1; i++) { for(i_=0; i_<=df.nvars-1;i_++) { x[i_] = xy[i,i_]; } dfprocess(df, x, ref y, _params); if( df.nclasses>1 ) { // // classification-specific code // k = (int)Math.Round(xy[i,df.nvars]); tmpi = 0; for(j=1; j<=df.nclasses-1; j++) { if( (double)(y[j])>(double)(y[tmpi]) ) { tmpi = j; } } if( (double)(y[k])!=(double)(0) ) { result = result-Math.Log(y[k]); } else { result = result-Math.Log(math.minrealnumber); } } } result = result/npoints; return result; } /************************************************************************* RMS error on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: root mean square error. Its meaning for regression task is obvious. As for classification task, RMS error means error when estimating posterior probabilities. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfrmserror(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; double[] x = new double[0]; double[] y = new double[0]; int i = 0; int j = 0; int k = 0; int tmpi = 0; int i_ = 0; x = new double[df.nvars-1+1]; y = new double[df.nclasses-1+1]; result = 0; for(i=0; i<=npoints-1; i++) { for(i_=0; i_<=df.nvars-1;i_++) { x[i_] = xy[i,i_]; } dfprocess(df, x, ref y, _params); if( df.nclasses>1 ) { // // classification-specific code // k = (int)Math.Round(xy[i,df.nvars]); tmpi = 0; for(j=1; j<=df.nclasses-1; j++) { if( (double)(y[j])>(double)(y[tmpi]) ) { tmpi = j; } } for(j=0; j<=df.nclasses-1; j++) { if( j==k ) { result = result+math.sqr(y[j]-1); } else { result = result+math.sqr(y[j]); } } } else { // // regression-specific code // result = result+math.sqr(y[0]-xy[i,df.nvars]); } } result = Math.Sqrt(result/(npoints*df.nclasses)); return result; } /************************************************************************* Average error on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: Its meaning for regression task is obvious. As for classification task, it means average error when estimating posterior probabilities. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfavgerror(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; double[] x = new double[0]; double[] y = new double[0]; int i = 0; int j = 0; int k = 0; int i_ = 0; x = new double[df.nvars-1+1]; y = new double[df.nclasses-1+1]; result = 0; for(i=0; i<=npoints-1; i++) { for(i_=0; i_<=df.nvars-1;i_++) { x[i_] = xy[i,i_]; } dfprocess(df, x, ref y, _params); if( df.nclasses>1 ) { // // classification-specific code // k = (int)Math.Round(xy[i,df.nvars]); for(j=0; j<=df.nclasses-1; j++) { if( j==k ) { result = result+Math.Abs(y[j]-1); } else { result = result+Math.Abs(y[j]); } } } else { // // regression-specific code // result = result+Math.Abs(y[0]-xy[i,df.nvars]); } } result = result/(npoints*df.nclasses); return result; } /************************************************************************* Average relative error on the test set INPUT PARAMETERS: DF - decision forest model XY - test set NPoints - test set size RESULT: Its meaning for regression task is obvious. As for classification task, it means average relative error when estimating posterior probability of belonging to the correct class. -- ALGLIB -- Copyright 16.02.2009 by Bochkanov Sergey *************************************************************************/ public static double dfavgrelerror(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { double result = 0; double[] x = new double[0]; double[] y = new double[0]; int relcnt = 0; int i = 0; int j = 0; int k = 0; int i_ = 0; x = new double[df.nvars-1+1]; y = new double[df.nclasses-1+1]; result = 0; relcnt = 0; for(i=0; i<=npoints-1; i++) { for(i_=0; i_<=df.nvars-1;i_++) { x[i_] = xy[i,i_]; } dfprocess(df, x, ref y, _params); if( df.nclasses>1 ) { // // classification-specific code // k = (int)Math.Round(xy[i,df.nvars]); for(j=0; j<=df.nclasses-1; j++) { if( j==k ) { result = result+Math.Abs(y[j]-1); relcnt = relcnt+1; } } } else { // // regression-specific code // if( (double)(xy[i,df.nvars])!=(double)(0) ) { result = result+Math.Abs((y[0]-xy[i,df.nvars])/xy[i,df.nvars]); relcnt = relcnt+1; } } } if( relcnt>0 ) { result = result/relcnt; } return result; } /************************************************************************* Copying of DecisionForest strucure INPUT PARAMETERS: DF1 - original OUTPUT PARAMETERS: DF2 - copy -- ALGLIB -- Copyright 13.02.2009 by Bochkanov Sergey *************************************************************************/ public static void dfcopy(decisionforest df1, decisionforest df2, alglib.xparams _params) { int i = 0; int bufsize = 0; int i_ = 0; if( df1.forestformat==dfuncompressedv0 ) { df2.forestformat = df1.forestformat; df2.nvars = df1.nvars; df2.nclasses = df1.nclasses; df2.ntrees = df1.ntrees; df2.bufsize = df1.bufsize; df2.trees = new double[df1.bufsize]; for(i_=0; i_<=df1.bufsize-1;i_++) { df2.trees[i_] = df1.trees[i_]; } dfcreatebuffer(df2, df2.buffer, _params); return; } if( df1.forestformat==dfcompressedv0 ) { df2.forestformat = df1.forestformat; df2.usemantissa8 = df1.usemantissa8; df2.nvars = df1.nvars; df2.nclasses = df1.nclasses; df2.ntrees = df1.ntrees; bufsize = alglib.ap.len(df1.trees8); df2.trees8 = new byte[bufsize]; for(i=0; i<=bufsize-1; i++) { df2.trees8[i] = unchecked((byte)(df1.trees8[i])); } dfcreatebuffer(df2, df2.buffer, _params); return; } alglib.ap.assert(false, "DFCopy: unexpected forest format"); } /************************************************************************* Serializer: allocation -- ALGLIB -- Copyright 14.03.2011 by Bochkanov Sergey *************************************************************************/ public static void dfalloc(alglib.serializer s, decisionforest forest, alglib.xparams _params) { if( forest.forestformat==dfuncompressedv0 ) { s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); apserv.allocrealarray(s, forest.trees, forest.bufsize, _params); return; } if( forest.forestformat==dfcompressedv0 ) { s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); s.alloc_entry(); s.alloc_byte_array(forest.trees8); return; } alglib.ap.assert(false, "DFAlloc: unexpected forest format"); } /************************************************************************* Serializer: serialization -- ALGLIB -- Copyright 14.03.2011 by Bochkanov Sergey *************************************************************************/ public static void dfserialize(alglib.serializer s, decisionforest forest, alglib.xparams _params) { if( forest.forestformat==dfuncompressedv0 ) { s.serialize_int(scodes.getrdfserializationcode(_params)); s.serialize_int(dfuncompressedv0); s.serialize_int(forest.nvars); s.serialize_int(forest.nclasses); s.serialize_int(forest.ntrees); s.serialize_int(forest.bufsize); apserv.serializerealarray(s, forest.trees, forest.bufsize, _params); return; } if( forest.forestformat==dfcompressedv0 ) { s.serialize_int(scodes.getrdfserializationcode(_params)); s.serialize_int(forest.forestformat); s.serialize_bool(forest.usemantissa8); s.serialize_int(forest.nvars); s.serialize_int(forest.nclasses); s.serialize_int(forest.ntrees); s.serialize_byte_array(forest.trees8); return; } alglib.ap.assert(false, "DFSerialize: unexpected forest format"); } /************************************************************************* Serializer: unserialization -- ALGLIB -- Copyright 14.03.2011 by Bochkanov Sergey *************************************************************************/ public static void dfunserialize(alglib.serializer s, decisionforest forest, alglib.xparams _params) { int i0 = 0; int forestformat = 0; bool processed = new bool(); // // check correctness of header // i0 = s.unserialize_int(); alglib.ap.assert(i0==scodes.getrdfserializationcode(_params), "DFUnserialize: stream header corrupted"); // // Read forest // forestformat = s.unserialize_int(); processed = false; if( forestformat==dfuncompressedv0 ) { // // Unserialize data // forest.forestformat = forestformat; forest.nvars = s.unserialize_int(); forest.nclasses = s.unserialize_int(); forest.ntrees = s.unserialize_int(); forest.bufsize = s.unserialize_int(); apserv.unserializerealarray(s, ref forest.trees, _params); processed = true; } if( forestformat==dfcompressedv0 ) { // // Unserialize data // forest.forestformat = forestformat; forest.usemantissa8 = s.unserialize_bool(); forest.nvars = s.unserialize_int(); forest.nclasses = s.unserialize_int(); forest.ntrees = s.unserialize_int(); forest.trees8 = s.unserialize_byte_array(); processed = true; } alglib.ap.assert(processed, "DFUnserialize: unexpected forest format"); // // Prepare buffer // dfcreatebuffer(forest, forest.buffer, _params); } /************************************************************************* This subroutine builds random decision forest. --------- DEPRECATED VERSION! USE DECISION FOREST BUILDER OBJECT --------- -- ALGLIB -- Copyright 19.02.2009 by Bochkanov Sergey *************************************************************************/ public static void dfbuildrandomdecisionforest(double[,] xy, int npoints, int nvars, int nclasses, int ntrees, double r, ref int info, decisionforest df, dfreport rep, alglib.xparams _params) { int samplesize = 0; info = 0; if( (double)(r)<=(double)(0) || (double)(r)>(double)(1) ) { info = -1; return; } samplesize = Math.Max((int)Math.Round(r*npoints), 1); dfbuildinternal(xy, npoints, nvars, nclasses, ntrees, samplesize, Math.Max(nvars/2, 1), dfusestrongsplits+dfuseevs, ref info, df, rep, _params); } /************************************************************************* This subroutine builds random decision forest. --------- DEPRECATED VERSION! USE DECISION FOREST BUILDER OBJECT --------- -- ALGLIB -- Copyright 19.02.2009 by Bochkanov Sergey *************************************************************************/ public static void dfbuildrandomdecisionforestx1(double[,] xy, int npoints, int nvars, int nclasses, int ntrees, int nrndvars, double r, ref int info, decisionforest df, dfreport rep, alglib.xparams _params) { int samplesize = 0; info = 0; if( (double)(r)<=(double)(0) || (double)(r)>(double)(1) ) { info = -1; return; } if( nrndvars<=0 || nrndvars>nvars ) { info = -1; return; } samplesize = Math.Max((int)Math.Round(r*npoints), 1); dfbuildinternal(xy, npoints, nvars, nclasses, ntrees, samplesize, nrndvars, dfusestrongsplits+dfuseevs, ref info, df, rep, _params); } public static void dfbuildinternal(double[,] xy, int npoints, int nvars, int nclasses, int ntrees, int samplesize, int nfeatures, int flags, ref int info, decisionforest df, dfreport rep, alglib.xparams _params) { decisionforestbuilder builder = new decisionforestbuilder(); int i = 0; info = 0; // // Test for inputs // if( (((((npoints<1 || samplesize<1) || samplesize>npoints) || nvars<1) || nclasses<1) || ntrees<1) || nfeatures<1 ) { info = -1; return; } if( nclasses>1 ) { for(i=0; i<=npoints-1; i++) { if( (int)Math.Round(xy[i,nvars])<0 || (int)Math.Round(xy[i,nvars])>=nclasses ) { info = -2; return; } } } info = 1; dfbuildercreate(builder, _params); dfbuildersetdataset(builder, xy, npoints, nvars, nclasses, _params); dfbuildersetsubsampleratio(builder, (double)samplesize/(double)npoints, _params); dfbuildersetrndvars(builder, nfeatures, _params); dfbuilderbuildrandomforest(builder, ntrees, df, rep, _params); } /************************************************************************* Builds a range of random trees [TreeIdx0,TreeIdx1) using decision forest algorithm. Tree index is used to seed per-tree RNG. -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static void buildrandomtree(decisionforestbuilder s, int treeidx0, int treeidx1, alglib.xparams _params) { int treeidx = 0; int i = 0; int j = 0; int npoints = 0; int nvars = 0; int nclasses = 0; hqrnd.hqrndstate rs = new hqrnd.hqrndstate(); dfworkbuf workbuf = null; dfvotebuf votebuf = null; dftreebuf treebuf = null; int treesize = 0; int varstoselect = 0; int workingsetsize = 0; double meanloss = 0; // // Perform parallelization // if( treeidx1-treeidx0>1 ) { if( _trypexec_buildrandomtree(s,treeidx0,treeidx1, _params) ) { return; } j = (treeidx1-treeidx0)/2; buildrandomtree(s, treeidx0, treeidx0+j, _params); buildrandomtree(s, treeidx0+j, treeidx1, _params); return; } else { alglib.ap.assert(treeidx1-treeidx0==1, "RDF: integrity check failed"); treeidx = treeidx0; } // // Prepare // npoints = s.npoints; nvars = s.nvars; nclasses = s.nclasses; if( s.rdfglobalseed>0 ) { hqrnd.hqrndseed(s.rdfglobalseed, 1+treeidx, rs, _params); } else { hqrnd.hqrndseed(math.randominteger(30000), 1+treeidx, rs, _params); } // // Retrieve buffers. // alglib.smp.ae_shared_pool_retrieve(s.workpool, ref workbuf); alglib.smp.ae_shared_pool_retrieve(s.votepool, ref votebuf); // // Prepare everything for tree construction. // alglib.ap.assert(workbuf.trnsize>=1, "DForest: integrity check failed (34636)"); alglib.ap.assert(workbuf.oobsize>=0, "DForest: integrity check failed (45745)"); alglib.ap.assert(workbuf.trnsize+workbuf.oobsize==npoints, "DForest: integrity check failed (89415)"); workingsetsize = -1; workbuf.varpoolsize = 0; for(i=0; i<=nvars-1; i++) { if( (double)(s.dsmin[i])!=(double)(s.dsmax[i]) ) { workbuf.varpool[workbuf.varpoolsize] = i; apserv.inc(ref workbuf.varpoolsize, _params); } } workingsetsize = workbuf.varpoolsize; alglib.ap.assert(workingsetsize>=0, "DForest: integrity check failed (73f5)"); for(i=0; i<=npoints-1; i++) { workbuf.tmp0i[i] = i; } for(i=0; i<=workbuf.trnsize-1; i++) { j = hqrnd.hqrnduniformi(rs, npoints-i, _params); apserv.swapelementsi(workbuf.tmp0i, i, i+j, _params); workbuf.trnset[i] = workbuf.tmp0i[i]; if( nclasses>1 ) { workbuf.trnlabelsi[i] = s.dsival[workbuf.tmp0i[i]]; } else { workbuf.trnlabelsr[i] = s.dsrval[workbuf.tmp0i[i]]; } if( s.neediobmatrix ) { s.iobmatrix[treeidx,workbuf.trnset[i]] = true; } } for(i=0; i<=workbuf.oobsize-1; i++) { j = workbuf.tmp0i[workbuf.trnsize+i]; workbuf.oobset[i] = j; if( nclasses>1 ) { workbuf.ooblabelsi[i] = s.dsival[j]; } else { workbuf.ooblabelsr[i] = s.dsrval[j]; } } varstoselect = (int)Math.Round(Math.Sqrt(nvars)); if( (double)(s.rdfvars)>(double)(0) ) { varstoselect = (int)Math.Round(s.rdfvars); } if( (double)(s.rdfvars)<(double)(0) ) { varstoselect = (int)Math.Round(-(nvars*s.rdfvars)); } varstoselect = Math.Max(varstoselect, 1); varstoselect = Math.Min(varstoselect, nvars); // // Perform recurrent construction // if( s.rdfimportance==needtrngini ) { meanloss = meannrms2(nclasses, workbuf.trnlabelsi, workbuf.trnlabelsr, 0, workbuf.trnsize, workbuf.trnlabelsi, workbuf.trnlabelsr, 0, workbuf.trnsize, ref workbuf.tmpnrms2, _params); } else { meanloss = meannrms2(nclasses, workbuf.trnlabelsi, workbuf.trnlabelsr, 0, workbuf.trnsize, workbuf.ooblabelsi, workbuf.ooblabelsr, 0, workbuf.oobsize, ref workbuf.tmpnrms2, _params); } treesize = 1; buildrandomtreerec(s, workbuf, workingsetsize, varstoselect, workbuf.treebuf, votebuf, rs, 0, workbuf.trnsize, 0, workbuf.oobsize, meanloss, meanloss, ref treesize, _params); workbuf.treebuf[0] = treesize; // // Store tree // alglib.smp.ae_shared_pool_retrieve(s.treefactory, ref treebuf); treebuf.treebuf = new double[treesize]; for(i=0; i<=treesize-1; i++) { treebuf.treebuf[i] = workbuf.treebuf[i]; } treebuf.treeidx = treeidx; alglib.smp.ae_shared_pool_recycle(s.treepool, ref treebuf); // // Return other buffers to appropriate pools // alglib.smp.ae_shared_pool_recycle(s.workpool, ref workbuf); alglib.smp.ae_shared_pool_recycle(s.votepool, ref votebuf); // // Update progress indicator // apserv.threadunsafeincby(ref s.rdfprogress, npoints, _params); } /************************************************************************* Serial stub for GPL edition. *************************************************************************/ public static bool _trypexec_buildrandomtree(decisionforestbuilder s, int treeidx0, int treeidx1, alglib.xparams _params) { return false; } /************************************************************************* Recurrent tree construction function using caller-allocated buffers and caller-initialized RNG. Following iterms are processed: * items [Idx0,Idx1) of WorkBuf.TrnSet * items [OOBIdx0, OOBIdx1) of WorkBuf.OOBSet TreeSize on input must be 1 (header element of the tree), on output it contains size of the tree. OOBLoss on input must contain value of MeanNRMS2(...) computed for entire dataset. Variables from #0 to #WorkingSet-1 from WorkBuf.VarPool are used (for block algorithm: blocks, not vars) -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static void buildrandomtreerec(decisionforestbuilder s, dfworkbuf workbuf, int workingset, int varstoselect, double[] treebuf, dfvotebuf votebuf, hqrnd.hqrndstate rs, int idx0, int idx1, int oobidx0, int oobidx1, double meanloss, double topmostmeanloss, ref int treesize, alglib.xparams _params) { int npoints = 0; int nclasses = 0; int i = 0; int j = 0; int j0 = 0; double v = 0; bool labelsaresame = new bool(); int offs = 0; int varbest = 0; double splitbest = 0; int i1 = 0; int i2 = 0; int idxtrn = 0; int idxoob = 0; double meanloss0 = 0; double meanloss1 = 0; alglib.ap.assert(s.dstype==0, "not supported skbdgfsi!"); alglib.ap.assert(idx01 ) { labelsaresame = true; for(i=0; i<=nclasses-1; i++) { workbuf.classpriors[i] = 0; } j0 = workbuf.trnlabelsi[idx0]; for(i=idx0; i<=idx1-1; i++) { j = workbuf.trnlabelsi[i]; workbuf.classpriors[j] = workbuf.classpriors[j]+1; labelsaresame = labelsaresame && j0==j; } } else { labelsaresame = false; } // // Leaf node // if( idx1-idx0==1 || labelsaresame ) { if( nclasses==1 ) { outputleaf(s, workbuf, treebuf, votebuf, idx0, idx1, oobidx0, oobidx1, ref treesize, workbuf.trnlabelsr[idx0], _params); } else { outputleaf(s, workbuf, treebuf, votebuf, idx0, idx1, oobidx0, oobidx1, ref treesize, workbuf.trnlabelsi[idx0], _params); } return; } // // Non-leaf node. // Investigate possible splits. // alglib.ap.assert(s.rdfalgo==0, "BuildRandomForest: unexpected algo"); choosecurrentsplitdense(s, workbuf, ref workingset, varstoselect, rs, idx0, idx1, ref varbest, ref splitbest, _params); if( varbest<0 ) { // // No good split was found; make leaf (label is randomly chosen) and exit. // if( nclasses>1 ) { v = workbuf.trnlabelsi[idx0+hqrnd.hqrnduniformi(rs, idx1-idx0, _params)]; } else { v = workbuf.trnlabelsr[idx0+hqrnd.hqrnduniformi(rs, idx1-idx0, _params)]; } outputleaf(s, workbuf, treebuf, votebuf, idx0, idx1, oobidx0, oobidx1, ref treesize, v, _params); return; } // // Good split WAS found, we can perform it: // * first, we split training set // * then, we similarly split OOB set // alglib.ap.assert(s.dstype==0, "not supported 54bfdh"); offs = npoints*varbest; i1 = idx0; i2 = idx1-1; while( i1<=i2 ) { // // Reorder indexes so that left partition is in [Idx0..I1), // and right partition is in [I2+1..Idx1) // if( workbuf.bestvals[i1]=splitbest ) { i2 = i2-1; continue; } j = workbuf.trnset[i1]; workbuf.trnset[i1] = workbuf.trnset[i2]; workbuf.trnset[i2] = j; if( nclasses>1 ) { j = workbuf.trnlabelsi[i1]; workbuf.trnlabelsi[i1] = workbuf.trnlabelsi[i2]; workbuf.trnlabelsi[i2] = j; } else { v = workbuf.trnlabelsr[i1]; workbuf.trnlabelsr[i1] = workbuf.trnlabelsr[i2]; workbuf.trnlabelsr[i2] = v; } i1 = i1+1; i2 = i2-1; } alglib.ap.assert(i1==i2+1, "BuildRandomTreeRec: integrity check failed (45rds3)"); idxtrn = i1; if( oobidx0=splitbest ) { i2 = i2-1; continue; } j = workbuf.oobset[i1]; workbuf.oobset[i1] = workbuf.oobset[i2]; workbuf.oobset[i2] = j; if( nclasses>1 ) { j = workbuf.ooblabelsi[i1]; workbuf.ooblabelsi[i1] = workbuf.ooblabelsi[i2]; workbuf.ooblabelsi[i2] = j; } else { v = workbuf.ooblabelsr[i1]; workbuf.ooblabelsr[i1] = workbuf.ooblabelsr[i2]; workbuf.ooblabelsr[i2] = v; } i1 = i1+1; i2 = i2-1; } alglib.ap.assert(i1==i2+1, "BuildRandomTreeRec: integrity check failed (643fs3)"); idxoob = i1; } else { idxoob = oobidx0; } // // Compute estimates of NRMS2 loss over TRN or OOB subsets, update Gini importances // if( s.rdfimportance==needtrngini ) { meanloss0 = meannrms2(nclasses, workbuf.trnlabelsi, workbuf.trnlabelsr, idx0, idxtrn, workbuf.trnlabelsi, workbuf.trnlabelsr, idx0, idxtrn, ref workbuf.tmpnrms2, _params); meanloss1 = meannrms2(nclasses, workbuf.trnlabelsi, workbuf.trnlabelsr, idxtrn, idx1, workbuf.trnlabelsi, workbuf.trnlabelsr, idxtrn, idx1, ref workbuf.tmpnrms2, _params); } else { meanloss0 = meannrms2(nclasses, workbuf.trnlabelsi, workbuf.trnlabelsr, idx0, idxtrn, workbuf.ooblabelsi, workbuf.ooblabelsr, oobidx0, idxoob, ref workbuf.tmpnrms2, _params); meanloss1 = meannrms2(nclasses, workbuf.trnlabelsi, workbuf.trnlabelsr, idxtrn, idx1, workbuf.ooblabelsi, workbuf.ooblabelsr, idxoob, oobidx1, ref workbuf.tmpnrms2, _params); } votebuf.giniimportances[varbest] = votebuf.giniimportances[varbest]+(meanloss-(meanloss0+meanloss1))/(topmostmeanloss+1.0e-20); // // Generate tree node and subtrees (recursively) // treebuf[treesize] = varbest; treebuf[treesize+1] = splitbest; i = treesize; treesize = treesize+innernodewidth; buildrandomtreerec(s, workbuf, workingset, varstoselect, treebuf, votebuf, rs, idx0, idxtrn, oobidx0, idxoob, meanloss0, topmostmeanloss, ref treesize, _params); treebuf[i+2] = treesize; buildrandomtreerec(s, workbuf, workingset, varstoselect, treebuf, votebuf, rs, idxtrn, idx1, idxoob, oobidx1, meanloss1, topmostmeanloss, ref treesize, _params); } /************************************************************************* Estimates permutation variable importance ratings for a range of dataset points. Initial call to this function should span entire range of the dataset, [Idx0,Idx1)=[0,NPoints), because function performs initialization of some internal structures when called with these arguments. -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static void estimatevariableimportance(decisionforestbuilder s, int sessionseed, decisionforest df, int ntrees, dfreport rep, alglib.xparams _params) { int npoints = 0; int nvars = 0; int nclasses = 0; int nperm = 0; int i = 0; int j = 0; int k = 0; dfvotebuf vote = null; double[] tmpr0 = new double[0]; double[] tmpr1 = new double[0]; int[] tmpi0 = new int[0]; double[] losses = new double[0]; dfpermimpbuf permseed = new dfpermimpbuf(); dfpermimpbuf permresult = null; alglib.smp.shared_pool permpool = new alglib.smp.shared_pool(); double nopermloss = 0; double totalpermloss = 0; hqrnd.hqrndstate varimprs = new hqrnd.hqrndstate(); npoints = s.npoints; nvars = s.nvars; nclasses = s.nclasses; // // No importance rating // if( s.rdfimportance==0 ) { return; } // // Gini importance // if( s.rdfimportance==needtrngini || s.rdfimportance==needoobgini ) { // // Merge OOB Gini importances computed during tree generation // alglib.smp.ae_shared_pool_first_recycled(s.votepool, ref vote); while( vote!=null ) { for(i=0; i<=nvars-1; i++) { rep.varimportances[i] = rep.varimportances[i]+vote.giniimportances[i]/ntrees; } alglib.smp.ae_shared_pool_next_recycled(s.votepool, ref vote); } for(i=0; i<=nvars-1; i++) { rep.varimportances[i] = apserv.boundval(rep.varimportances[i], 0, 1, _params); } // // Compute topvars[] array // tmpr0 = new double[nvars]; for(j=0; j<=nvars-1; j++) { tmpr0[j] = -rep.varimportances[j]; rep.topvars[j] = j; } tsort.tagsortfasti(ref tmpr0, ref rep.topvars, ref tmpr1, ref tmpi0, nvars, _params); return; } // // Permutation importance // if( s.rdfimportance==needpermutation ) { alglib.ap.assert(df.forestformat==dfuncompressedv0, "EstimateVariableImportance: integrity check failed (ff)"); alglib.ap.assert(alglib.ap.rows(s.iobmatrix)>=ntrees && alglib.ap.cols(s.iobmatrix)>=npoints, "EstimateVariableImportance: integrity check failed (IOB)"); // // Generate packed representation of the shuffle which is applied to all variables // // Ideally we want to apply different permutations to different variables, // i.e. we have to generate and store NPoints*NVars random numbers. // However due to performance and memory restrictions we prefer to use compact // representation: // * we store one "reference" permutation P_ref in VarImpShuffle2[0:NPoints-1] // * a permutation P_j applied to variable J is obtained by circularly shifting // elements in P_ref by VarImpShuffle2[NPoints+J] // hqrnd.hqrndseed(sessionseed, 1117, varimprs, _params); apserv.ivectorsetlengthatleast(ref s.varimpshuffle2, npoints+nvars, _params); for(i=0; i<=npoints-1; i++) { s.varimpshuffle2[i] = i; } for(i=0; i<=npoints-2; i++) { j = i+hqrnd.hqrnduniformi(varimprs, npoints-i, _params); k = s.varimpshuffle2[i]; s.varimpshuffle2[i] = s.varimpshuffle2[j]; s.varimpshuffle2[j] = k; } for(i=0; i<=nvars-1; i++) { s.varimpshuffle2[npoints+i] = hqrnd.hqrnduniformi(varimprs, npoints, _params); } // // Prepare buffer object, seed pool // nperm = nvars+2; permseed.losses = new double[nperm]; for(j=0; j<=nperm-1; j++) { permseed.losses[j] = 0; } permseed.yv = new double[nperm*nclasses]; permseed.xraw = new double[nvars]; permseed.xdist = new double[nvars]; permseed.xcur = new double[nvars]; permseed.targety = new double[nclasses]; permseed.startnodes = new int[nvars]; permseed.y = new double[nclasses]; alglib.smp.ae_shared_pool_set_seed(permpool, permseed); // // Recursively split subset and process (using parallel capabilities, if possible) // estimatepermutationimportances(s, df, ntrees, permpool, 0, npoints, _params); // // Merge results // losses = new double[nperm]; for(j=0; j<=nperm-1; j++) { losses[j] = 1.0e-20; } alglib.smp.ae_shared_pool_first_recycled(permpool, ref permresult); while( permresult!=null ) { for(j=0; j<=nperm-1; j++) { losses[j] = losses[j]+permresult.losses[j]; } alglib.smp.ae_shared_pool_next_recycled(permpool, ref permresult); } // // Compute importances // nopermloss = losses[nvars+1]; totalpermloss = losses[nvars]; for(i=0; i<=nvars-1; i++) { rep.varimportances[i] = 1-nopermloss/totalpermloss-(1-losses[i]/totalpermloss); rep.varimportances[i] = apserv.boundval(rep.varimportances[i], 0, 1, _params); } // // Compute topvars[] array // tmpr0 = new double[nvars]; for(j=0; j<=nvars-1; j++) { tmpr0[j] = -rep.varimportances[j]; rep.topvars[j] = j; } tsort.tagsortfasti(ref tmpr0, ref rep.topvars, ref tmpr1, ref tmpi0, nvars, _params); return; } alglib.ap.assert(false, "EstimateVariableImportance: unexpected importance type"); } /************************************************************************* Serial stub for GPL edition. *************************************************************************/ public static bool _trypexec_estimatevariableimportance(decisionforestbuilder s, int sessionseed, decisionforest df, int ntrees, dfreport rep, alglib.xparams _params) { return false; } /************************************************************************* Estimates permutation variable importance ratings for a range of dataset points. Initial call to this function should span entire range of the dataset, [Idx0,Idx1)=[0,NPoints), because function performs initialization of some internal structures when called with these arguments. -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static void estimatepermutationimportances(decisionforestbuilder s, decisionforest df, int ntrees, alglib.smp.shared_pool permpool, int idx0, int idx1, alglib.xparams _params) { int npoints = 0; int nvars = 0; int nclasses = 0; int nperm = 0; int i = 0; int j = 0; int k = 0; double v = 0; int treeroot = 0; int nodeoffs = 0; double prediction = 0; int varidx = 0; int oobcounts = 0; int srcidx = 0; dfpermimpbuf permimpbuf = null; npoints = s.npoints; nvars = s.nvars; nclasses = s.nclasses; alglib.ap.assert(df.forestformat==dfuncompressedv0, "EstimateVariableImportance: integrity check failed (ff)"); alglib.ap.assert((idx0>=0 && idx0<=idx1) && idx1<=npoints, "EstimateVariableImportance: integrity check failed (idx)"); alglib.ap.assert(alglib.ap.rows(s.iobmatrix)>=ntrees && alglib.ap.cols(s.iobmatrix)>=npoints, "EstimateVariableImportance: integrity check failed (IOB)"); // // Perform parallelization if batch is too large // if( idx1-idx0>permutationimportancebatchsize ) { if( _trypexec_estimatepermutationimportances(s,df,ntrees,permpool,idx0,idx1, _params) ) { return; } j = (idx1-idx0)/2; estimatepermutationimportances(s, df, ntrees, permpool, idx0, idx0+j, _params); estimatepermutationimportances(s, df, ntrees, permpool, idx0+j, idx1, _params); return; } // // Retrieve buffer object from pool // alglib.smp.ae_shared_pool_retrieve(permpool, ref permimpbuf); // // Process range of points [idx0,idx1) // nperm = nvars+2; for(i=idx0; i<=idx1-1; i++) { alglib.ap.assert(s.dstype==0, "EstimateVariableImportance: unexpected dataset type"); for(j=0; j<=nvars-1; j++) { permimpbuf.xraw[j] = s.dsdata[j*npoints+i]; srcidx = s.varimpshuffle2[(i+s.varimpshuffle2[npoints+j])%npoints]; permimpbuf.xdist[j] = s.dsdata[j*npoints+srcidx]; } if( nclasses>1 ) { for(j=0; j<=nclasses-1; j++) { permimpbuf.targety[j] = 0; } permimpbuf.targety[s.dsival[i]] = 1; } else { permimpbuf.targety[0] = s.dsrval[i]; } // // Process all trees, for each tree compute NPerm losses corresponding // to various permutations of variable values // for(j=0; j<=nperm*nclasses-1; j++) { permimpbuf.yv[j] = 0; } oobcounts = 0; treeroot = 0; for(k=0; k<=ntrees-1; k++) { if( !s.iobmatrix[k,i] ) { // // Process original (unperturbed) point and analyze path from the // tree root to the final leaf. Output prediction to RawPrediction. // // Additionally, for each variable in [0,NVars-1] save offset of // the first split on this variable. It allows us to quickly compute // tree decision when perturbation does not change decision path. // alglib.ap.assert(df.forestformat==dfuncompressedv0, "EstimateVariableImportance: integrity check failed (ff)"); nodeoffs = treeroot+1; for(j=0; j<=nvars-1; j++) { permimpbuf.startnodes[j] = -1; } prediction = 0; while( true ) { if( (double)(df.trees[nodeoffs])==(double)(-1) ) { prediction = df.trees[nodeoffs+1]; break; } j = (int)Math.Round(df.trees[nodeoffs]); if( permimpbuf.startnodes[j]<0 ) { permimpbuf.startnodes[j] = nodeoffs; } if( permimpbuf.xraw[j]1 ) { j = (int)Math.Round(prediction); permimpbuf.yv[varidx*nclasses+j] = permimpbuf.yv[varidx*nclasses+j]+1; } else { permimpbuf.yv[varidx] = permimpbuf.yv[varidx]+prediction; } // // Save loss for all variables being perturbed (XDist). // This loss is used as a reference loss when we compute R-squared. // varidx = nvars; for(j=0; j<=nclasses-1; j++) { permimpbuf.y[j] = 0; } dfprocessinternaluncompressed(df, treeroot, treeroot+1, permimpbuf.xdist, ref permimpbuf.y, _params); for(j=0; j<=nclasses-1; j++) { permimpbuf.yv[varidx*nclasses+j] = permimpbuf.yv[varidx*nclasses+j]+permimpbuf.y[j]; } // // Compute losses for variable #VarIdx being perturbed. Quite an often decision // process does not actually depend on the variable #VarIdx (path from the tree // root does not include splits on this variable). In such cases we perform // quick exit from the loop with precomputed value. // for(j=0; j<=nvars-1; j++) { permimpbuf.xcur[j] = permimpbuf.xraw[j]; } for(varidx=0; varidx<=nvars-1; varidx++) { if( permimpbuf.startnodes[varidx]>=0 ) { // // Path from tree root to the final leaf involves split on variable #VarIdx. // Restart computation from the position first split on #VarIdx. // alglib.ap.assert(df.forestformat==dfuncompressedv0, "EstimateVariableImportance: integrity check failed (ff)"); permimpbuf.xcur[varidx] = permimpbuf.xdist[varidx]; nodeoffs = permimpbuf.startnodes[varidx]; while( true ) { if( (double)(df.trees[nodeoffs])==(double)(-1) ) { if( nclasses>1 ) { j = (int)Math.Round(df.trees[nodeoffs+1]); permimpbuf.yv[varidx*nclasses+j] = permimpbuf.yv[varidx*nclasses+j]+1; } else { permimpbuf.yv[varidx] = permimpbuf.yv[varidx]+df.trees[nodeoffs+1]; } break; } j = (int)Math.Round(df.trees[nodeoffs]); if( permimpbuf.xcur[j]1 ) { j = (int)Math.Round(prediction); permimpbuf.yv[varidx*nclasses+j] = permimpbuf.yv[varidx*nclasses+j]+1; } else { permimpbuf.yv[varidx] = permimpbuf.yv[varidx]+prediction; } } } // // update OOB counter // apserv.inc(ref oobcounts, _params); } treeroot = treeroot+(int)Math.Round(df.trees[treeroot]); } // // Now YV[] stores NPerm versions of the forest output for various permutations of variable values. // Update losses. // for(j=0; j<=nperm-1; j++) { for(k=0; k<=nclasses-1; k++) { permimpbuf.yv[j*nclasses+k] = permimpbuf.yv[j*nclasses+k]/apserv.coalesce(oobcounts, 1, _params); } v = 0; for(k=0; k<=nclasses-1; k++) { v = v+math.sqr(permimpbuf.yv[j*nclasses+k]-permimpbuf.targety[k]); } permimpbuf.losses[j] = permimpbuf.losses[j]+v; } // // Update progress indicator // apserv.threadunsafeincby(ref s.rdfprogress, ntrees, _params); } // // Recycle buffer object with updated Losses[] field // alglib.smp.ae_shared_pool_recycle(permpool, ref permimpbuf); } /************************************************************************* Serial stub for GPL edition. *************************************************************************/ public static bool _trypexec_estimatepermutationimportances(decisionforestbuilder s, decisionforest df, int ntrees, alglib.smp.shared_pool permpool, int idx0, int idx1, alglib.xparams _params) { return false; } /************************************************************************* Sets report fields to their default values -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static void cleanreport(decisionforestbuilder s, dfreport rep, alglib.xparams _params) { int i = 0; rep.relclserror = 0; rep.avgce = 0; rep.rmserror = 0; rep.avgerror = 0; rep.avgrelerror = 0; rep.oobrelclserror = 0; rep.oobavgce = 0; rep.oobrmserror = 0; rep.oobavgerror = 0; rep.oobavgrelerror = 0; rep.topvars = new int[s.nvars]; rep.varimportances = new double[s.nvars]; for(i=0; i<=s.nvars-1; i++) { rep.topvars[i] = i; rep.varimportances[i] = 0; } } /************************************************************************* This function returns NRMS2 loss (sum of squared residuals) for a constant- output model: * model output is a mean over TRN set being passed (for classification problems - NClasses-dimensional vector of class probabilities) * model is evaluated over TST set being passed, with L2 loss being returned Input parameters: NClasses - ">1" for classification, "=1" for regression TrnLabelsI - training set labels, class indexes (for NClasses>1) TrnLabelsR - training set output values (for NClasses=1) TrnIdx0, TrnIdx1 - a range [Idx0,Idx1) of elements in LabelsI/R is considered TstLabelsI - training set labels, class indexes (for NClasses>1) TstLabelsR - training set output values (for NClasses=1) TstIdx0, TstIdx1 - a range [Idx0,Idx1) of elements in LabelsI/R is considered TmpI - temporary array, reallocated as needed Result: sum of squared residuals; for NClasses>=2 it coincides with Gini impurity times (Idx1-Idx0) Following fields of WorkBuf are used as temporaries: * TmpMeanNRMS2 -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static double meannrms2(int nclasses, int[] trnlabelsi, double[] trnlabelsr, int trnidx0, int trnidx1, int[] tstlabelsi, double[] tstlabelsr, int tstidx0, int tstidx1, ref int[] tmpi, alglib.xparams _params) { double result = 0; int i = 0; int k = 0; int ntrn = 0; int ntst = 0; double v = 0; double vv = 0; double invntrn = 0; double pitrn = 0; double nitst = 0; alglib.ap.assert(trnidx0<=trnidx1, "MeanNRMS2: integrity check failed (8754)"); alglib.ap.assert(tstidx0<=tstidx1, "MeanNRMS2: integrity check failed (8754)"); result = 0; ntrn = trnidx1-trnidx0; ntst = tstidx1-tstidx0; if( ntrn==0 || ntst==0 ) { return result; } invntrn = 1.0/ntrn; if( nclasses>1 ) { // // Classification problem // apserv.ivectorsetlengthatleast(ref tmpi, 2*nclasses, _params); for(i=0; i<=2*nclasses-1; i++) { tmpi[i] = 0; } for(i=trnidx0; i<=trnidx1-1; i++) { k = trnlabelsi[i]; tmpi[k] = tmpi[k]+1; } for(i=tstidx0; i<=tstidx1-1; i++) { k = tstlabelsi[i]; tmpi[k+nclasses] = tmpi[k+nclasses]+1; } for(i=0; i<=nclasses-1; i++) { pitrn = tmpi[i]*invntrn; nitst = tmpi[i+nclasses]; result = result+nitst*(1-pitrn)*(1-pitrn); result = result+(ntst-nitst)*pitrn*pitrn; } } else { // // regression-specific code // v = 0; for(i=trnidx0; i<=trnidx1-1; i++) { v = v+trnlabelsr[i]; } v = v*invntrn; for(i=tstidx0; i<=tstidx1-1; i++) { vv = tstlabelsr[i]-v; result = result+vv*vv; } } return result; } /************************************************************************* This function is a part of the recurrent tree construction function; it selects variable for splitting according to current tree construction algorithm. Note: modifies VarsInPool, may decrease it if some variables become non-informative and leave the pool. -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static void choosecurrentsplitdense(decisionforestbuilder s, dfworkbuf workbuf, ref int varsinpool, int varstoselect, hqrnd.hqrndstate rs, int idx0, int idx1, ref int varbest, ref double splitbest, alglib.xparams _params) { int npoints = 0; double errbest = 0; int varstried = 0; int varcur = 0; bool valuesaresame = new bool(); int offs = 0; double split = 0; int i = 0; double v = 0; double v0 = 0; double currms = 0; int info = 0; varbest = 0; splitbest = 0; alglib.ap.assert(s.dstype==0, "sparsity is not supported 4terg!"); alglib.ap.assert(s.rdfalgo==0, "BuildRandomTreeRec: integrity check failed (1657)"); alglib.ap.assert(idx00 && (varbest<0 || (double)(currms)<=(double)(errbest)) ) { errbest = currms; varbest = varcur; splitbest = split; for(i=idx0; i<=idx1-1; i++) { workbuf.bestvals[i] = workbuf.curvals[i]; } } // // Next iteration // varstried = varstried+1; } } /************************************************************************* This function performs split on some specific dense variable whose values are stored in WorkBuf.CurVals[Idx0,Idx1) and labels are stored in WorkBuf.TrnLabelsR/I[Idx0,Idx1). It returns split value and associated RMS error. It is responsibility of the caller to make sure that variable has at least two distinct values, i.e. it is possible to make a split. Precomputed values of following fields of WorkBuf are used: * ClassPriors Following fields of WorkBuf are used as temporaries: * ClassTotals0,1,01 * Tmp0I, Tmp1I, Tmp0R, Tmp1R, Tmp2R, Tmp3R -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static void evaluatedensesplit(decisionforestbuilder s, dfworkbuf workbuf, hqrnd.hqrndstate rs, int splitvar, int idx0, int idx1, ref int info, ref double split, ref double rms, alglib.xparams _params) { int nclasses = 0; int i = 0; int j = 0; int k0 = 0; int k1 = 0; double v = 0; double v0 = 0; double v1 = 0; double v2 = 0; int sl = 0; int sr = 0; info = 0; split = 0; rms = 0; alglib.ap.assert(idx01 ) { // // Classification problem // for(j=0; j<=nclasses-1; j++) { workbuf.classtotals0[j] = 0; } sl = 0; for(i=idx0; i<=idx1-1; i++) { if( workbuf.curvals[i]1 ) { for(i=0; i<=idx1-idx0-1; i++) { workbuf.tmp0r[i] = workbuf.curvals[idx0+i]; workbuf.tmp0i[i] = workbuf.trnlabelsi[idx0+i]; } classifiersplit(s, workbuf, ref workbuf.tmp0r, ref workbuf.tmp0i, idx1-idx0, rs, ref info, ref split, ref rms, ref workbuf.tmp1r, ref workbuf.tmp1i, _params); } else { for(i=0; i<=idx1-idx0-1; i++) { workbuf.tmp0r[i] = workbuf.curvals[idx0+i]; workbuf.tmp1r[i] = workbuf.trnlabelsr[idx0+i]; } regressionsplit(s, workbuf, ref workbuf.tmp0r, ref workbuf.tmp1r, idx1-idx0, ref info, ref split, ref rms, ref workbuf.tmp2r, ref workbuf.tmp3r, _params); } } } /************************************************************************* Classifier split *************************************************************************/ private static void classifiersplit(decisionforestbuilder s, dfworkbuf workbuf, ref double[] x, ref int[] c, int n, hqrnd.hqrndstate rs, ref int info, ref double threshold, ref double e, ref double[] sortrbuf, ref int[] sortibuf, alglib.xparams _params) { int i = 0; int j = 0; int k = 0; int n0 = 0; int n0prev = 0; double v = 0; int advanceby = 0; double rms = 0; int k0 = 0; int k1 = 0; double v0 = 0; double v1 = 0; int nclasses = 0; double vmin = 0; double vmax = 0; info = 0; threshold = 0; e = 0; alglib.ap.assert((s.rdfsplitstrength==0 || s.rdfsplitstrength==1) || s.rdfsplitstrength==2, "RDF: unexpected split type at ClassifierSplit()"); nclasses = s.nclasses; advanceby = 1; if( n>=20 ) { advanceby = Math.Max(2, (int)Math.Round(n*0.05)); } info = -1; threshold = 0; e = math.maxrealnumber; // // Random split // if( s.rdfsplitstrength==0 ) { // // Evaluate minimum, maximum and randomly selected values // vmin = x[0]; vmax = x[0]; for(i=1; i<=n-1; i++) { v = x[i]; if( vvmax ) { vmax = v; } } if( (double)(vmin)==(double)(vmax) ) { return; } v = x[hqrnd.hqrnduniformi(rs, n, _params)]; if( (double)(v)==(double)(vmin) ) { v = vmax; } // // Calculate RMS error associated with the split // for(i=0; i<=nclasses-1; i++) { workbuf.classtotals0[i] = 0; } n0 = 0; for(i=0; i<=n-1; i++) { if( x[i]0 && n00 && n00 ) { e = Math.Sqrt(e/(nclasses*n)); } return; } alglib.ap.assert(false, "RDF: ClassifierSplit(), critical error"); } /************************************************************************* Regression model split *************************************************************************/ private static void regressionsplit(decisionforestbuilder s, dfworkbuf workbuf, ref double[] x, ref double[] y, int n, ref int info, ref double threshold, ref double e, ref double[] sortrbuf, ref double[] sortrbuf2, alglib.xparams _params) { int i = 0; double vmin = 0; double vmax = 0; double bnd01 = 0; double bnd12 = 0; double bnd23 = 0; int total0 = 0; int total1 = 0; int total2 = 0; int total3 = 0; int cnt0 = 0; int cnt1 = 0; int cnt2 = 0; int cnt3 = 0; int n0 = 0; int advanceby = 0; double v = 0; double v0 = 0; double v1 = 0; double rms = 0; int n0prev = 0; int k0 = 0; int k1 = 0; info = 0; threshold = 0; e = 0; advanceby = 1; if( n>=20 ) { advanceby = Math.Max(2, (int)Math.Round(n*0.05)); } // // Sort data // Quick check for degeneracy // tsort.tagsortfastr(ref x, ref y, ref sortrbuf, ref sortrbuf2, n, _params); v = 0.5*(x[0]+x[n-1]); if( !((double)(x[0])<(double)(v) && (double)(v)<(double)(x[n-1])) ) { info = -1; threshold = x[n-1]; e = math.maxrealnumber; return; } // // Prepare initial split. // Evaluate current split, prepare next one, repeat. // vmin = y[0]; vmax = y[0]; for(i=1; i<=n-1; i++) { v = y[i]; if( vvmax ) { vmax = v; } } bnd12 = 0.5*(vmin+vmax); bnd01 = 0.5*(vmin+bnd12); bnd23 = 0.5*(vmax+bnd12); total0 = 0; total1 = 0; total2 = 0; total3 = 0; for(i=0; i<=n-1; i++) { v = y[i]; if( v0 ) { e = Math.Sqrt(e/(4*n)); } } /************************************************************************* Returns split: either deterministic split at the middle of [A,B], or randomly chosen split. It is guaranteed that A0 ) { hqrnd.hqrndseed(s.rdfglobalseed, 3532, rs, _params); } else { hqrnd.hqrndseed(math.randominteger(30000), 3532, rs, _params); } // // Generic processing // alglib.ap.assert(npoints>=1, "BuildRandomForest: integrity check failed"); apserv.rvectorsetlengthatleast(ref s.dsmin, nvars, _params); apserv.rvectorsetlengthatleast(ref s.dsmax, nvars, _params); apserv.bvectorsetlengthatleast(ref s.dsbinary, nvars, _params); for(i=0; i<=nvars-1; i++) { v0 = s.dsdata[i*npoints+0]; v1 = s.dsdata[i*npoints+0]; for(j=1; j<=npoints-1; j++) { v = s.dsdata[i*npoints+j]; if( vv1 ) { v1 = v; } } s.dsmin[i] = v0; s.dsmax[i] = v1; alglib.ap.assert((double)(v0)<=(double)(v1), "BuildRandomForest: strange integrity check failure"); isbinary = true; for(j=0; j<=npoints-1; j++) { v = s.dsdata[i*npoints+j]; isbinary = isbinary && (v==v0 || v==v1); } s.dsbinary[i] = isbinary; } if( nclasses==1 ) { s.dsravg = 0; for(i=0; i<=npoints-1; i++) { s.dsravg = s.dsravg+s.dsrval[i]; } s.dsravg = s.dsravg/npoints; } else { apserv.ivectorsetlengthatleast(ref s.dsctotals, nclasses, _params); for(i=0; i<=nclasses-1; i++) { s.dsctotals[i] = 0; } for(i=0; i<=npoints-1; i++) { s.dsctotals[s.dsival[i]] = s.dsctotals[s.dsival[i]]+1; } } } /************************************************************************* This function merges together trees generated during training and outputs it to the decision forest. INPUT PARAMETERS: S - decision forest builder object NTrees - NTrees>=1, number of trees to train OUTPUT PARAMETERS: DF - decision forest Rep - report -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static void mergetrees(decisionforestbuilder s, decisionforest df, alglib.xparams _params) { int i = 0; int cursize = 0; int offs = 0; dftreebuf tree = null; int[] treesizes = new int[0]; int[] treeoffsets = new int[0]; df.forestformat = dfuncompressedv0; df.nvars = s.nvars; df.nclasses = s.nclasses; df.bufsize = 0; df.ntrees = 0; // // Determine trees count // alglib.smp.ae_shared_pool_first_recycled(s.treepool, ref tree); while( tree!=null ) { df.ntrees = df.ntrees+1; alglib.smp.ae_shared_pool_next_recycled(s.treepool, ref tree); } alglib.ap.assert(df.ntrees>0, "MergeTrees: integrity check failed, zero trees count"); // // Determine individual tree sizes and total buffer size // treesizes = new int[df.ntrees]; for(i=0; i<=df.ntrees-1; i++) { treesizes[i] = -1; } alglib.smp.ae_shared_pool_first_recycled(s.treepool, ref tree); while( tree!=null ) { alglib.ap.assert(tree.treeidx>=0 && tree.treeidx0, "MergeTrees: integrity check failed (wrong TreeSize)"); } // // Determine offsets for individual trees in output buffer // treeoffsets = new int[df.ntrees]; treeoffsets[0] = 0; for(i=1; i<=df.ntrees-1; i++) { treeoffsets[i] = treeoffsets[i-1]+treesizes[i-1]; } // // Output trees // // NOTE: since ALGLIB 3.16.0 trees are sorted by tree index prior to // output (necessary for variable importance estimation), that's // why we need array of tree offsets // df.trees = new double[df.bufsize]; alglib.smp.ae_shared_pool_first_recycled(s.treepool, ref tree); while( tree!=null ) { cursize = (int)Math.Round(tree.treebuf[0]); offs = treeoffsets[tree.treeidx]; for(i=0; i<=cursize-1; i++) { df.trees[offs+i] = tree.treebuf[i]; } alglib.smp.ae_shared_pool_next_recycled(s.treepool, ref tree); } } /************************************************************************* This function post-processes voting array and calculates TRN and OOB errors. INPUT PARAMETERS: S - decision forest builder object NTrees - number of trees in the forest Buf - possibly preallocated vote buffer, its contents is overwritten by this function OUTPUT PARAMETERS: Rep - report fields corresponding to errors are updated -- ALGLIB -- Copyright 21.05.2018 by Bochkanov Sergey *************************************************************************/ private static void processvotingresults(decisionforestbuilder s, int ntrees, dfvotebuf buf, dfreport rep, alglib.xparams _params) { dfvotebuf vote = null; int nvars = 0; int nclasses = 0; int npoints = 0; int i = 0; int j = 0; int k = 0; int k1 = 0; double v = 0; int avgrelcnt = 0; int oobavgrelcnt = 0; npoints = s.npoints; nvars = s.nvars; nclasses = s.nclasses; alglib.ap.assert(npoints>0, "DFOREST: integrity check failed"); alglib.ap.assert(nvars>0, "DFOREST: integrity check failed"); alglib.ap.assert(nclasses>0, "DFOREST: integrity check failed"); // // Prepare vote buffer // apserv.rvectorsetlengthatleast(ref buf.trntotals, npoints*nclasses, _params); apserv.rvectorsetlengthatleast(ref buf.oobtotals, npoints*nclasses, _params); for(i=0; i<=npoints*nclasses-1; i++) { buf.trntotals[i] = 0; buf.oobtotals[i] = 0; } apserv.ivectorsetlengthatleast(ref buf.trncounts, npoints, _params); apserv.ivectorsetlengthatleast(ref buf.oobcounts, npoints, _params); for(i=0; i<=npoints-1; i++) { buf.trncounts[i] = 0; buf.oobcounts[i] = 0; } // // Merge voting arrays // alglib.smp.ae_shared_pool_first_recycled(s.votepool, ref vote); while( vote!=null ) { for(i=0; i<=npoints*nclasses-1; i++) { buf.trntotals[i] = buf.trntotals[i]+vote.trntotals[i]+vote.oobtotals[i]; buf.oobtotals[i] = buf.oobtotals[i]+vote.oobtotals[i]; } for(i=0; i<=npoints-1; i++) { buf.trncounts[i] = buf.trncounts[i]+vote.trncounts[i]+vote.oobcounts[i]; buf.oobcounts[i] = buf.oobcounts[i]+vote.oobcounts[i]; } alglib.smp.ae_shared_pool_next_recycled(s.votepool, ref vote); } for(i=0; i<=npoints-1; i++) { v = 1/apserv.coalesce(buf.trncounts[i], 1, _params); for(j=0; j<=nclasses-1; j++) { buf.trntotals[i*nclasses+j] = buf.trntotals[i*nclasses+j]*v; } v = 1/apserv.coalesce(buf.oobcounts[i], 1, _params); for(j=0; j<=nclasses-1; j++) { buf.oobtotals[i*nclasses+j] = buf.oobtotals[i*nclasses+j]*v; } } // // Use aggregated voting data to output error metrics // avgrelcnt = 0; oobavgrelcnt = 0; rep.rmserror = 0; rep.avgerror = 0; rep.avgrelerror = 0; rep.relclserror = 0; rep.avgce = 0; rep.oobrmserror = 0; rep.oobavgerror = 0; rep.oobavgrelerror = 0; rep.oobrelclserror = 0; rep.oobavgce = 0; for(i=0; i<=npoints-1; i++) { if( nclasses>1 ) { // // classification-specific code // k = s.dsival[i]; for(j=0; j<=nclasses-1; j++) { v = buf.trntotals[i*nclasses+j]; if( j==k ) { rep.avgce = rep.avgce-Math.Log(apserv.coalesce(v, math.minrealnumber, _params)); rep.rmserror = rep.rmserror+math.sqr(v-1); rep.avgerror = rep.avgerror+Math.Abs(v-1); rep.avgrelerror = rep.avgrelerror+Math.Abs(v-1); apserv.inc(ref avgrelcnt, _params); } else { rep.rmserror = rep.rmserror+math.sqr(v); rep.avgerror = rep.avgerror+Math.Abs(v); } v = buf.oobtotals[i*nclasses+j]; if( j==k ) { rep.oobavgce = rep.oobavgce-Math.Log(apserv.coalesce(v, math.minrealnumber, _params)); rep.oobrmserror = rep.oobrmserror+math.sqr(v-1); rep.oobavgerror = rep.oobavgerror+Math.Abs(v-1); rep.oobavgrelerror = rep.oobavgrelerror+Math.Abs(v-1); apserv.inc(ref oobavgrelcnt, _params); } else { rep.oobrmserror = rep.oobrmserror+math.sqr(v); rep.oobavgerror = rep.oobavgerror+Math.Abs(v); } } // // Classification errors are handled separately // k1 = 0; for(j=1; j<=nclasses-1; j++) { if( buf.trntotals[i*nclasses+j]>buf.trntotals[i*nclasses+k1] ) { k1 = j; } } if( k1!=k ) { rep.relclserror = rep.relclserror+1; } k1 = 0; for(j=1; j<=nclasses-1; j++) { if( buf.oobtotals[i*nclasses+j]>buf.oobtotals[i*nclasses+k1] ) { k1 = j; } } if( k1!=k ) { rep.oobrelclserror = rep.oobrelclserror+1; } } else { // // regression-specific code // v = buf.trntotals[i]-s.dsrval[i]; rep.rmserror = rep.rmserror+math.sqr(v); rep.avgerror = rep.avgerror+Math.Abs(v); if( (double)(s.dsrval[i])!=(double)(0) ) { rep.avgrelerror = rep.avgrelerror+Math.Abs(v/s.dsrval[i]); avgrelcnt = avgrelcnt+1; } v = buf.oobtotals[i]-s.dsrval[i]; rep.oobrmserror = rep.oobrmserror+math.sqr(v); rep.oobavgerror = rep.oobavgerror+Math.Abs(v); if( (double)(s.dsrval[i])!=(double)(0) ) { rep.oobavgrelerror = rep.oobavgrelerror+Math.Abs(v/s.dsrval[i]); oobavgrelcnt = oobavgrelcnt+1; } } } rep.relclserror = rep.relclserror/npoints; rep.rmserror = Math.Sqrt(rep.rmserror/(npoints*nclasses)); rep.avgerror = rep.avgerror/(npoints*nclasses); rep.avgrelerror = rep.avgrelerror/apserv.coalesce(avgrelcnt, 1, _params); rep.oobrelclserror = rep.oobrelclserror/npoints; rep.oobrmserror = Math.Sqrt(rep.oobrmserror/(npoints*nclasses)); rep.oobavgerror = rep.oobavgerror/(npoints*nclasses); rep.oobavgrelerror = rep.oobavgrelerror/apserv.coalesce(oobavgrelcnt, 1, _params); } /************************************************************************* This function performs binary compression of decision forest, using either 8-bit mantissa (a bit more compact representation) or 16-bit mantissa for splits and regression outputs. Forest is compressed in-place. Return value is a compression factor. -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ private static double binarycompression(decisionforest df, bool usemantissa8, alglib.xparams _params) { double result = 0; int size8 = 0; int size8i = 0; int offssrc = 0; int offsdst = 0; int i = 0; int[] dummyi = new int[0]; int maxrawtreesize = 0; int[] compressedsizes = new int[0]; // // Quick exit if already compressed // if( df.forestformat==dfcompressedv0 ) { result = 1; return result; } // // Check that source format is supported // alglib.ap.assert(df.forestformat==dfuncompressedv0, "BinaryCompression: unexpected forest format"); // // Compute sizes of uncompressed and compressed trees. // size8 = 0; offssrc = 0; maxrawtreesize = 0; for(i=0; i<=df.ntrees-1; i++) { size8i = computecompressedsizerec(df, usemantissa8, offssrc, offssrc+1, dummyi, false, _params); size8 = size8+computecompresseduintsize(size8i, _params)+size8i; maxrawtreesize = Math.Max(maxrawtreesize, (int)Math.Round(df.trees[offssrc])); offssrc = offssrc+(int)Math.Round(df.trees[offssrc]); } result = (double)(8*alglib.ap.len(df.trees))/(double)(size8+1); // // Allocate memory and perform compression // df.trees8 = new byte[size8]; compressedsizes = new int[maxrawtreesize]; offssrc = 0; offsdst = 0; for(i=0; i<=df.ntrees-1; i++) { // // Call compressed size evaluator one more time, now saving subtree sizes into temporary array // size8i = computecompressedsizerec(df, usemantissa8, offssrc, offssrc+1, compressedsizes, true, _params); // // Output tree header (length in bytes) // streamuint(df.trees8, ref offsdst, size8i, _params); // // Compress recursively // compressrec(df, usemantissa8, offssrc, offssrc+1, compressedsizes, df.trees8, ref offsdst, _params); // // Next tree // offssrc = offssrc+(int)Math.Round(df.trees[offssrc]); } alglib.ap.assert(offsdst==size8, "BinaryCompression: integrity check failed (stream length)"); // // Finalize forest conversion, clear previously allocated memory // df.forestformat = dfcompressedv0; df.usemantissa8 = usemantissa8; df.trees = new double[0]; return result; } /************************************************************************* This function returns exact number of bytes required to store compressed version of the tree starting at location TreeBase. PARAMETERS: DF - decision forest UseMantissa8 - whether 8-bit or 16-bit mantissas are used to store floating point numbers TreeRoot - root of the specific tree being stored (offset in DF.Trees) TreePos - position within tree (first location in the tree is TreeRoot+1) CompressedSizes - not referenced if SaveCompressedSizes is False; otherwise, values computed by this function for specific values of TreePos are stored to CompressedSizes[TreePos-TreeRoot] (other elements of the array are not referenced). This array must be preallocated by caller. -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ private static int computecompressedsizerec(decisionforest df, bool usemantissa8, int treeroot, int treepos, int[] compressedsizes, bool savecompressedsizes, alglib.xparams _params) { int result = 0; int jmponbranch = 0; int child0size = 0; int child1size = 0; int fpwidth = 0; if( usemantissa8 ) { fpwidth = 2; } else { fpwidth = 3; } // // Leaf or split? // if( (double)(df.trees[treepos])==(double)(-1) ) { // // Leaf // result = computecompresseduintsize(2*df.nvars, _params); if( df.nclasses==1 ) { result = result+fpwidth; } else { result = result+computecompresseduintsize((int)Math.Round(df.trees[treepos+1]), _params); } } else { // // Split // jmponbranch = (int)Math.Round(df.trees[treepos+2]); child0size = computecompressedsizerec(df, usemantissa8, treeroot, treepos+innernodewidth, compressedsizes, savecompressedsizes, _params); child1size = computecompressedsizerec(df, usemantissa8, treeroot, treeroot+jmponbranch, compressedsizes, savecompressedsizes, _params); if( child0size<=child1size ) { // // Child #0 comes first because it is shorter // result = computecompresseduintsize((int)Math.Round(df.trees[treepos]), _params); result = result+fpwidth; result = result+computecompresseduintsize(child0size, _params); } else { // // Child #1 comes first because it is shorter // result = computecompresseduintsize((int)Math.Round(df.trees[treepos])+df.nvars, _params); result = result+fpwidth; result = result+computecompresseduintsize(child1size, _params); } result = result+child0size+child1size; } // // Do we have to save compressed sizes? // if( savecompressedsizes ) { alglib.ap.assert(treepos-treeroot=VAL then BRANCH0 else BRANCH1" // * stream value used for splitting // * stream children #0 and #1 // streamuint(buf, ref dstoffs, varidx+df.nvars, _params); streamfloat(buf, usemantissa8, ref dstoffs, splitval, _params); streamuint(buf, ref dstoffs, child1size, _params); compressrec(df, usemantissa8, treeroot, treeroot+jmponbranch, compressedsizes, buf, ref dstoffs, _params); compressrec(df, usemantissa8, treeroot, treepos+innernodewidth, compressedsizes, buf, ref dstoffs, _params); } } // // Integrity check at the end // alglib.ap.assert(dstoffs-dstoffsold==compressedsizes[treepos-treeroot], "CompressRec: integrity check failed (compressed size at leaf)"); } /************************************************************************* This function returns exact number of bytes required to store compressed unsigned integer number (negative arguments result in assertion being generated). -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ private static int computecompresseduintsize(int v, alglib.xparams _params) { int result = 0; alglib.ap.assert(v>=0); result = 1; while( v>=128 ) { v = v/128; result = result+1; } return result; } /************************************************************************* This function stores compressed unsigned integer number (negative arguments result in assertion being generated) to byte array at location Offs and increments Offs by number of bytes being stored. -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ private static void streamuint(byte[] buf, ref int offs, int v, alglib.xparams _params) { int v0 = 0; alglib.ap.assert(v>=0); while( true ) { // // Save 7 least significant bits of V, use 8th bit as a flag which // tells us whether subsequent 7-bit packages will be sent. // v0 = v%128; if( v>=128 ) { v0 = v0+128; } buf[offs] = unchecked((byte)(v0)); offs = offs+1; v = v/128; if( v==0 ) { break; } } } /************************************************************************* This function reads compressed unsigned integer number from byte array starting at location Offs and increments Offs by number of bytes being read. -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ private static int unstreamuint(byte[] buf, ref int offs, alglib.xparams _params) { int result = 0; int v0 = 0; int p = 0; result = 0; p = 1; while( true ) { // // Rad 7 bits of V, use 8th bit as a flag which tells us whether // subsequent 7-bit packages will be received. // v0 = buf[offs]; offs = offs+1; result = result+v0%128*p; if( v0<128 ) { break; } p = p*128; } return result; } /************************************************************************* This function stores compressed floating point number to byte array at location Offs and increments Offs by number of bytes being stored. Either 8-bit mantissa or 16-bit mantissa is used. The exponent is always 7 bits of exponent + sign. Values which do not fit into exponent range are truncated to fit. -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ private static void streamfloat(byte[] buf, bool usemantissa8, ref int offs, double v, alglib.xparams _params) { int signbit = 0; int e = 0; int m = 0; double twopow30 = 0; double twopowm30 = 0; double twopow10 = 0; double twopowm10 = 0; alglib.ap.assert(math.isfinite(v), "StreamFloat: V is not finite number"); // // Special case: zero // if( v==0.0 ) { if( usemantissa8 ) { buf[offs+0] = unchecked((byte)(0)); buf[offs+1] = unchecked((byte)(0)); offs = offs+2; } else { buf[offs+0] = unchecked((byte)(0)); buf[offs+1] = unchecked((byte)(0)); buf[offs+2] = unchecked((byte)(0)); offs = offs+3; } return; } // // Handle sign // signbit = 0; if( v<0.0 ) { v = -v; signbit = 128; } // // Compute exponent // twopow30 = 1073741824; twopow10 = 1024; twopowm30 = 1.0/twopow30; twopowm10 = 1.0/twopow10; e = 0; while( v>=twopow30 ) { v = v*twopowm30; e = e+30; } while( v>=twopow10 ) { v = v*twopowm10; e = e+10; } while( v>=1.0 ) { v = v*0.5; e = e+1; } while( v=0.5 && v<1.0, "StreamFloat: integrity check failed"); // // Handle exponent underflow/overflow // if( e<-63 ) { signbit = 0; e = 0; v = 0; } if( e>63 ) { e = 63; v = 1.0; } // // Save to stream // if( usemantissa8 ) { m = (int)Math.Round(v*256); if( m==256 ) { m = m/2; e = Math.Min(e+1, 63); } buf[offs+0] = unchecked((byte)(e+64+signbit)); buf[offs+1] = unchecked((byte)(m)); offs = offs+2; } else { m = (int)Math.Round(v*65536); if( m==65536 ) { m = m/2; e = Math.Min(e+1, 63); } buf[offs+0] = unchecked((byte)(e+64+signbit)); buf[offs+1] = unchecked((byte)(m%256)); buf[offs+2] = unchecked((byte)(m/256)); offs = offs+3; } } /************************************************************************* This function reads compressed floating point number from the byte array starting from location Offs and increments Offs by number of bytes being read. Either 8-bit mantissa or 16-bit mantissa is used. The exponent is always 7 bits of exponent + sign. Values which do not fit into exponent range are truncated to fit. -- ALGLIB -- Copyright 22.07.2019 by Bochkanov Sergey *************************************************************************/ private static double unstreamfloat(byte[] buf, bool usemantissa8, ref int offs, alglib.xparams _params) { double result = 0; int e = 0; double v = 0; double inv256 = 0; // // Read from stream // inv256 = 1.0/256.0; if( usemantissa8 ) { e = buf[offs+0]; v = buf[offs+1]*inv256; offs = offs+2; } else { e = buf[offs+0]; v = (buf[offs+1]*inv256+buf[offs+2])*inv256; offs = offs+3; } // // Decode // if( e>128 ) { v = -v; e = e-128; } e = e-64; result = xfastpow(2, e, _params)*v; return result; } /************************************************************************* Classification error *************************************************************************/ private static int dfclserror(decisionforest df, double[,] xy, int npoints, alglib.xparams _params) { int result = 0; double[] x = new double[0]; double[] y = new double[0]; int i = 0; int j = 0; int k = 0; int tmpi = 0; int i_ = 0; if( df.nclasses<=1 ) { result = 0; return result; } x = new double[df.nvars-1+1]; y = new double[df.nclasses-1+1]; result = 0; for(i=0; i<=npoints-1; i++) { for(i_=0; i_<=df.nvars-1;i_++) { x[i_] = xy[i,i_]; } dfprocess(df, x, ref y, _params); k = (int)Math.Round(xy[i,df.nvars]); tmpi = 0; for(j=1; j<=df.nclasses-1; j++) { if( (double)(y[j])>(double)(y[tmpi]) ) { tmpi = j; } } if( tmpi!=k ) { result = result+1; } } return result; } /************************************************************************* Internal subroutine for processing one decision tree stored in uncompressed format starting at SubtreeRoot (this index points to the header of the tree, not its first node). First node being processed is located at NodeOffs. *************************************************************************/ private static void dfprocessinternaluncompressed(decisionforest df, int subtreeroot, int nodeoffs, double[] x, ref double[] y, alglib.xparams _params) { int idx = 0; alglib.ap.assert(df.forestformat==dfuncompressedv0, "DFProcessInternal: unexpected forest format"); // // Navigate through the tree // while( true ) { if( (double)(df.trees[nodeoffs])==(double)(-1) ) { if( df.nclasses==1 ) { y[0] = y[0]+df.trees[nodeoffs+1]; } else { idx = (int)Math.Round(df.trees[nodeoffs+1]); y[idx] = y[idx]+1; } break; } if( x[(int)Math.Round(df.trees[nodeoffs])]=splitval ) { offs = offs+jmplen; } } else { // // The split rule is "if VAR>=VAL then BRANCH0 else BRANCH1" // varidx = varidx-df.nvars; if( x[varidx]0 ) { if( n%2==0 ) { result = xfastpow(r, n/2, _params); result = result*result; } else { result = r*xfastpow(r, n-1, _params); } return result; } if( n==0 ) { result = 1; } if( n<0 ) { result = xfastpow(1/r, -n, _params); } return result; } } public class knn { /************************************************************************* Buffer object which is used to perform various requests (usually model inference) in the multithreaded mode (multiple threads working with same KNN object). This object should be created with KNNCreateBuffer(). *************************************************************************/ public class knnbuffer : apobject { public nearestneighbor.kdtreerequestbuffer treebuf; public double[] x; public double[] y; public int[] tags; public double[,] xy; public knnbuffer() { init(); } public override void init() { treebuf = new nearestneighbor.kdtreerequestbuffer(); x = new double[0]; y = new double[0]; tags = new int[0]; xy = new double[0,0]; } public override alglib.apobject make_copy() { knnbuffer _result = new knnbuffer(); _result.treebuf = (nearestneighbor.kdtreerequestbuffer)treebuf.make_copy(); _result.x = (double[])x.Clone(); _result.y = (double[])y.Clone(); _result.tags = (int[])tags.Clone(); _result.xy = (double[,])xy.Clone(); return _result; } }; /************************************************************************* A KNN builder object; this object encapsulates dataset and all related settings, it is used to create an actual instance of KNN model. *************************************************************************/ public class knnbuilder : apobject { public int dstype; public int npoints; public int nvars; public bool iscls; public int nout; public double[,] dsdata; public double[] dsrval; public int[] dsival; public int knnnrm; public knnbuilder() { init(); } public override void init() { dsdata = new double[0,0]; dsrval = new double[0]; dsival = new int[0]; } public override alglib.apobject make_copy() { knnbuilder _result = new knnbuilder(); _result.dstype = dstype; _result.npoints = npoints; _result.nvars = nvars; _result.iscls = iscls; _result.nout = nout; _result.dsdata = (double[,])dsdata.Clone(); _result.dsrval = (double[])dsrval.Clone(); _result.dsival = (int[])dsival.Clone(); _result.knnnrm = knnnrm; return _result; } }; /************************************************************************* KNN model, can be used for classification or regression *************************************************************************/ public class knnmodel : apobject { public int nvars; public int nout; public int k; public double eps; public bool iscls; public bool isdummy; public nearestneighbor.kdtree tree; public knnbuffer buffer; public knnmodel() { init(); } public override void init() { tree = new nearestneighbor.kdtree(); buffer = new knnbuffer(); } public override alglib.apobject make_copy() { knnmodel _result = new knnmodel(); _result.nvars = nvars; _result.nout = nout; _result.k = k; _result.eps = eps; _result.iscls = iscls; _result.isdummy = isdummy; _result.tree = (nearestneighbor.kdtree)tree.make_copy(); _result.buffer = (knnbuffer)buffer.make_copy(); return _result; } }; /************************************************************************* KNN training report. Following fields store training set errors: * relclserror - fraction of misclassified cases, [0,1] * avgce - average cross-entropy in bits per symbol * rmserror - root-mean-square error * avgerror - average error * avgrelerror - average relative error For classification problems: * RMS, AVG and AVGREL errors are calculated for posterior probabilities For regression problems: * RELCLS and AVGCE errors are zero *************************************************************************/ public class knnreport : apobject { public double relclserror; public double avgce; public double rmserror; public double avgerror; public double avgrelerror; public knnreport() { init(); } public override void init() { } public override alglib.apobject make_copy() { knnreport _result = new knnreport(); _result.relclserror = relclserror; _result.avgce = avgce; _result.rmserror = rmserror; _result.avgerror = avgerror; _result.avgrelerror = avgrelerror; return _result; } }; public const int knnfirstversion = 0; /************************************************************************* This function creates buffer structure which can be used to perform parallel KNN requests. KNN subpackage provides two sets of computing functions - ones which use internal buffer of KNN model (these functions are single-threaded because they use same buffer, which can not shared between threads), and ones which use external buffer. This function is used to initialize external buffer. INPUT PARAMETERS Model - KNN model which is associated with newly created buffer OUTPUT PARAMETERS Buf - external buffer. IMPORTANT: buffer object should be used only with model which was used to initialize buffer. Any attempt to use buffer with different object is dangerous - you may get integrity check failure (exception) because sizes of internal arrays do not fit to dimensions of the model structure. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knncreatebuffer(knnmodel model, knnbuffer buf, alglib.xparams _params) { if( !model.isdummy ) { nearestneighbor.kdtreecreaterequestbuffer(model.tree, buf.treebuf, _params); } buf.x = new double[model.nvars]; buf.y = new double[model.nout]; } /************************************************************************* This subroutine creates KNNBuilder object which is used to train KNN models. By default, new builder stores empty dataset and some reasonable default settings. At the very least, you should specify dataset prior to building KNN model. You can also tweak settings of the model construction algorithm (recommended, although default settings should work well). Following actions are mandatory: * calling knnbuildersetdataset() to specify dataset * calling knnbuilderbuildknnmodel() to build KNN model using current dataset and default settings Additionally, you may call: * knnbuildersetnorm() to change norm being used INPUT PARAMETERS: none OUTPUT PARAMETERS: S - KNN builder -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnbuildercreate(knnbuilder s, alglib.xparams _params) { // // Empty dataset // s.dstype = -1; s.npoints = 0; s.nvars = 0; s.iscls = false; s.nout = 1; // // Default training settings // s.knnnrm = 2; } /************************************************************************* Specifies regression problem (one or more continuous output variables are predicted). There also exists "classification" version of this function. This subroutine adds dense dataset to the internal storage of the builder object. Specifying your dataset in the dense format means that the dense version of the KNN construction algorithm will be invoked. INPUT PARAMETERS: S - KNN builder object XY - array[NPoints,NVars+NOut] (note: actual size can be larger, only leading part is used anyway), dataset: * first NVars elements of each row store values of the independent variables * next NOut elements store values of the dependent variables NPoints - number of rows in the dataset, NPoints>=1 NVars - number of independent variables, NVars>=1 NOut - number of dependent variables, NOut>=1 OUTPUT PARAMETERS: S - KNN builder -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnbuildersetdatasetreg(knnbuilder s, double[,] xy, int npoints, int nvars, int nout, alglib.xparams _params) { int i = 0; int j = 0; // // Check parameters // alglib.ap.assert(npoints>=1, "knnbuildersetdatasetreg: npoints<1"); alglib.ap.assert(nvars>=1, "knnbuildersetdatasetreg: nvars<1"); alglib.ap.assert(nout>=1, "knnbuildersetdatasetreg: nout<1"); alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "knnbuildersetdatasetreg: rows(xy)=nvars+nout, "knnbuildersetdatasetreg: cols(xy)=1 NVars - number of independent variables, NVars>=1 NClasses - number of classes, NClasses>=2 OUTPUT PARAMETERS: S - KNN builder -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnbuildersetdatasetcls(knnbuilder s, double[,] xy, int npoints, int nvars, int nclasses, alglib.xparams _params) { int i = 0; int j = 0; // // Check parameters // alglib.ap.assert(npoints>=1, "knnbuildersetdatasetcls: npoints<1"); alglib.ap.assert(nvars>=1, "knnbuildersetdatasetcls: nvars<1"); alglib.ap.assert(nclasses>=2, "knnbuildersetdatasetcls: nclasses<2"); alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "knnbuildersetdatasetcls: rows(xy)=nvars+1, "knnbuildersetdatasetcls: cols(xy)=0 && j1, Eps=0 corresponding to the "K nearest neighbors algorithm" * K>=1, Eps>0 corresponding to "approximate nearest neighbors algorithm" An approximate KNN is a good option for high-dimensional datasets (exact KNN works slowly when dimensions count grows). An ALGLIB implementation of kd-trees is used to perform k-nn searches. ! COMMERCIAL EDITION OF ALGLIB: ! ! Commercial Edition of ALGLIB includes following important improvements ! of this function: ! * high-performance native backend with same C# interface (C# version) ! * multithreading support (C++ and C# versions) ! ! We recommend you to read 'Working with commercial version' section of ! ALGLIB Reference Manual in order to find out how to use performance- ! related features provided by commercial edition of ALGLIB. INPUT PARAMETERS: S - KNN builder object K - number of neighbors to search for, K>=1 Eps - approximation factor: * Eps=0 means that exact kNN search is performed * Eps>0 means that (1+Eps)-approximate search is performed OUTPUT PARAMETERS: Model - KNN model Rep - report -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnbuilderbuildknnmodel(knnbuilder s, int k, double eps, knnmodel model, knnreport rep, alglib.xparams _params) { int i = 0; int j = 0; int nvars = 0; int nout = 0; int npoints = 0; bool iscls = new bool(); double[,] xy = new double[0,0]; int[] tags = new int[0]; npoints = s.npoints; nvars = s.nvars; nout = s.nout; iscls = s.iscls; // // Check settings // alglib.ap.assert(k>=1, "knnbuilderbuildknnmodel: k<1"); alglib.ap.assert(math.isfinite(eps) && (double)(eps)>=(double)(0), "knnbuilderbuildknnmodel: eps<0"); // // Prepare output // clearreport(rep, _params); model.nvars = nvars; model.nout = nout; model.iscls = iscls; model.k = k; model.eps = eps; model.isdummy = false; // // Quick exit for empty dataset // if( s.dstype==-1 ) { model.isdummy = true; return; } // // Build kd-tree // if( iscls ) { xy = new double[npoints, nvars+1]; tags = new int[npoints]; for(i=0; i<=npoints-1; i++) { for(j=0; j<=nvars-1; j++) { xy[i,j] = s.dsdata[i,j]; } xy[i,nvars] = s.dsival[i]; tags[i] = s.dsival[i]; } nearestneighbor.kdtreebuildtagged(xy, tags, npoints, nvars, 0, s.knnnrm, model.tree, _params); } else { xy = new double[npoints, nvars+nout]; for(i=0; i<=npoints-1; i++) { for(j=0; j<=nvars-1; j++) { xy[i,j] = s.dsdata[i,j]; } for(j=0; j<=nout-1; j++) { xy[i,nvars+j] = s.dsrval[i*nout+j]; } } nearestneighbor.kdtreebuild(xy, npoints, nvars, nout, s.knnnrm, model.tree, _params); } // // Build buffer // knncreatebuffer(model, model.buffer, _params); // // Report // knnallerrors(model, xy, npoints, rep, _params); } /************************************************************************* Changing search settings of KNN model. K and EPS parameters of KNN (AKNN) search are specified during model construction. However, plain KNN algorithm with Euclidean distance allows you to change them at any moment. NOTE: future versions of KNN model may support advanced versions of KNN, such as NCA or LMNN. It is possible that such algorithms won't allow you to change search settings on the fly. If you call this function for an algorithm which does not support on-the-fly changes, it will throw an exception. INPUT PARAMETERS: Model - KNN model K - K>=1, neighbors count EPS - accuracy of the EPS-approximate NN search. Set to 0.0, if you want to perform "classic" KNN search. Specify larger values if you need to speed-up high-dimensional KNN queries. OUTPUT PARAMETERS: nothing on success, exception on failure -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnrewritekeps(knnmodel model, int k, double eps, alglib.xparams _params) { alglib.ap.assert(k>=1, "knnrewritekeps: k<1"); alglib.ap.assert(math.isfinite(eps) && (double)(eps)>=(double)(0), "knnrewritekeps: eps<0"); model.k = k; model.eps = eps; } /************************************************************************* Inference using KNN model. See also knnprocess0(), knnprocessi() and knnclassify() for options with a bit more convenient interface. IMPORTANT: this function is thread-unsafe and modifies internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use knntsprocess() with independent thread-local buffers, if you need thread-safe evaluation. INPUT PARAMETERS: Model - KNN model X - input vector, array[0..NVars-1]. Y - possible preallocated buffer. Reused if long enough. OUTPUT PARAMETERS: Y - result. Regression estimate when solving regression task, vector of posterior probabilities for classification task. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnprocess(knnmodel model, double[] x, ref double[] y, alglib.xparams _params) { knntsprocess(model, model.buffer, x, ref y, _params); } /************************************************************************* This function returns first component of the inferred vector (i.e. one with index #0). It is a convenience wrapper for knnprocess() intended for either: * 1-dimensional regression problems * 2-class classification problems In the former case this function returns inference result as scalar, which is definitely more convenient that wrapping it as vector. In the latter case it returns probability of object belonging to class #0. If you call it for anything different from two cases above, it will work as defined, i.e. return y[0], although it is of less use in such cases. IMPORTANT: this function is thread-unsafe and modifies internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use knntsprocess() with independent thread-local buffers, if you need thread-safe evaluation. INPUT PARAMETERS: Model - KNN model X - input vector, array[0..NVars-1]. RESULT: Y[0] -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static double knnprocess0(knnmodel model, double[] x, alglib.xparams _params) { double result = 0; int i = 0; int nvars = 0; nvars = model.nvars; for(i=0; i<=nvars-1; i++) { model.buffer.x[i] = x[i]; } processinternal(model, model.buffer, _params); result = model.buffer.y[0]; return result; } /************************************************************************* This function returns most probable class number for an input X. It is same as calling knnprocess(model,x,y), then determining i=argmax(y[i]) and returning i. A class number in [0,NOut) range in returned for classification problems, -1 is returned when this function is called for regression problems. IMPORTANT: this function is thread-unsafe and modifies internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use knntsprocess() with independent thread-local buffers, if you need thread-safe evaluation. INPUT PARAMETERS: Model - KNN model X - input vector, array[0..NVars-1]. RESULT: class number, -1 for regression tasks -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static int knnclassify(knnmodel model, double[] x, alglib.xparams _params) { int result = 0; int i = 0; int nvars = 0; int nout = 0; if( !model.iscls ) { result = -1; return result; } nvars = model.nvars; nout = model.nout; for(i=0; i<=nvars-1; i++) { model.buffer.x[i] = x[i]; } processinternal(model, model.buffer, _params); result = 0; for(i=1; i<=nout-1; i++) { if( model.buffer.y[i]>model.buffer.y[result] ) { result = i; } } return result; } /************************************************************************* 'interactive' variant of knnprocess() for languages like Python which support constructs like "y = knnprocessi(model,x)" and interactive mode of the interpreter. This function allocates new array on each call, so it is significantly slower than its 'non-interactive' counterpart, but it is more convenient when you call it from command line. IMPORTANT: this function is thread-unsafe and may modify internal structures of the model! You can not use same model object for parallel evaluation from several threads. Use knntsprocess() with independent thread-local buffers if you need thread-safe evaluation. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnprocessi(knnmodel model, double[] x, ref double[] y, alglib.xparams _params) { y = new double[0]; knnprocess(model, x, ref y, _params); } /************************************************************************* Thread-safe procesing using external buffer for temporaries. This function is thread-safe (i.e . you can use same KNN model from multiple threads) as long as you use different buffer objects for different threads. INPUT PARAMETERS: Model - KNN model Buf - buffer object, must be allocated specifically for this model with knncreatebuffer(). X - input vector, array[NVars] OUTPUT PARAMETERS: Y - result, array[NOut]. Regression estimate when solving regression task, vector of posterior probabilities for a classification task. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knntsprocess(knnmodel model, knnbuffer buf, double[] x, ref double[] y, alglib.xparams _params) { int i = 0; int nvars = 0; int nout = 0; nvars = model.nvars; nout = model.nout; for(i=0; i<=nvars-1; i++) { buf.x[i] = x[i]; } processinternal(model, buf, _params); if( alglib.ap.len(y)=0 OUTPUT PARAMETERS: Rep - following fields are loaded with errors for both regression and classification models: * rep.rmserror - RMS error for the output * rep.avgerror - average error * rep.avgrelerror - average relative error following fields are set only for classification models, zero for regression ones: * relclserror - relative classification error, in [0,1] * avgce - average cross-entropy in bits per dataset entry NOTE: the cross-entropy metric is too unstable when used to evaluate KNN models (such models can report exactly zero probabilities), so we do not recommend using it. -- ALGLIB -- Copyright 15.02.2019 by Bochkanov Sergey *************************************************************************/ public static void knnallerrors(knnmodel model, double[,] xy, int npoints, knnreport rep, alglib.xparams _params) { knnbuffer buf = new knnbuffer(); double[] desiredy = new double[0]; double[] errbuf = new double[0]; int nvars = 0; int nout = 0; int ny = 0; bool iscls = new bool(); int i = 0; int j = 0; nvars = model.nvars; nout = model.nout; iscls = model.iscls; if( iscls ) { ny = 1; } else { ny = nout; } // // Check input // alglib.ap.assert(npoints>=0, "knnallerrors: npoints<0"); alglib.ap.assert(alglib.ap.rows(xy)>=npoints, "knnallerrors: rows(xy)=nvars+ny, "knnallerrors: cols(xy)=0 && j