Changeset 40319 for trunk/Ohana/src/opihi/cmd.data/nnet_train.c

Timestamp:

Jan 23, 2018, 10:11:50 AM (9 years ago)

Author:

eugene

Message:

working on nnet

File:

• trunk/Ohana/src/opihi/cmd.data/nnet_train.c (modified) (12 diffs)

trunk/Ohana/src/opihi/cmd.data/nnet_train.c

@@ -1 +1 @@
 # include "data.h"
-void sortseq (float *X, int *Y, int N);
+
+// these are local only
+float nnet_sigmoid (float value);
+void  nnet_sortseq (float *X, int *Y, int N);
+int   nnet_onelayer (Nnet *nnet, int L);
+void  nnet_reset_Nabla (Nnet *nnet);
+void  nnet_update_Nabla (Nnet *nnet);
+void  nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta);
+void  nnet_backprop (Nnet *nnet, Vector **inVec, Vector **outVec, int N);
+void  nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta);
 
 int nnet_train (int argc, char **argv) {
+
+  int N;
 
   int Nepoch = 10;
@@ -18 +29 @@
   }
   
+  float eta = 1.0; // XXX how do I set this? does it need to update?
+
   if (argc < 4) {
     gprint (GP_ERR, "USAGE: nnet train (nnet) [input] [input] ... [output] [output] ...\n");
@@ -97 +110 @@
     // generate a random sequence : used to select random mini batches
     for (int i = 0; i < Ntrial; i++) { seq[i] = i; rnd[i] = drand48(); }
-    sortseq (rnd, seq, Ntrial);
+    nnet_sortseq (rnd, seq, Ntrial);
 
     int Npass = Ntrial / Nmini;
@@ -116 +129 @@
 
 // this recreates 'update_mini_batch' from http://neuralnetworksanddeeplearning.com/chap1.html
-void nnet_descent_step (NNet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta) {
+void nnet_descent_step (Nnet *nnet, Vector **inVec, Vector **outVec, int *seq, int pass, int Nmini, float eta) {
 
   int Ntrial = inVec[0][0].Nelements;
@@ -122 +135 @@
   nnet_reset_Nabla (nnet);
 
-  for (i = 0; (i < Nmini) && (pass*Nmini + i < Ntrial); i++) {
+  for (int i = 0; (i < Nmini) && (pass*Nmini + i < Ntrial); i++) {
 
     // N is the element of the mini batch on which we are currently operating
-    N = seq[pass*Nmini + i];
+    int N = seq[pass*Nmini + i];
 
     // backprop generates a dNabla_b, dNabla_w pair for the element N of the input and output vectors
@@ -136 +149 @@
 }
 
-void nnet_backprop (NNet *nnet, Vector **inVec, Vector **outVec, int N) {
+void nnet_backprop (Nnet *nnet, Vector **inVec, Vector **outVec, int N) {
 
   // start with the input values
-
-  // store the input values for this row (trial) in the input value vector
   int Nlayer   = nnet[0].Nlayer;
-  int Ninput   = nnet[0].Nnodes[0];
-  int Noutput  = nnet[0].Nnodes[Nlayer - 1];  // number of input + output nodes
-
-  for (int j = 0; j < Ninput; j++) {
+
+  // store the input values for this row (trial) in the input vector "svalue[0]"
+  for (int j = 0; j < nnet[0].Nnodes[0]; j++) {
     nnet[0].svalue[0][j] = inVec[j][0].elements.Flt[N];
   }
@@ -153 +163 @@
   // activation = sigmoid (z)
 
+  // feedforward operates on the vector saved in svalue[0]
+  // feedforward saves the zvalues [w * input + bias], svalues [sigmoid(z)], sprimes [sigmoid'(z)] as it runs
   nnet_feedforward (nnet);
 
-  // backward pass (note for now these are vector operations:
-  for (L = Nlayer - 1; L > 0; L--) {
-
-    // sp is array of same size as zvalue for each layer
-    // sp = sigmoid_prime (nnet[0].zvalue[L]); sprime is precalculated in feedforward
+  // backward pass
+  for (int L = Nlayer - 1; L > 0; L--) {
 
     if (L == Nlayer - 1) {
       // starting point uses cost_derivative to compare last svalue set with truth output
-      // delta = cost_derivative(nnet[0].svalue[L], outVec, N) * sprime[L];
+      // delta = cost_derivative(svalue[L], output) * sprime[L];
       for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
-        nnet[0].delta[L][j] = cost_derivative(nnet[0].svalue[L][j], outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
+        // cost_derivative (svalue[L][j], output[j]) = svalue[L][j] - output[j]
+        // (because cost = (1/2)(svalue[L][j] - output[j])^2)
+        // NOTE: the (1/Npt) factor is pushed to the apply_Nabla step
+        // nnet[0].delta[L][j] = cost_derivative(nnet[0].svalue[L][j], outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
+        nnet[0].delta[L][j] = (nnet[0].svalue[L][j] - outVec[j][0].elements.Flt[N]) * nnet[0].sprime[L][j];
       }
     } else {
-      // delta = (delta DOT transpose(weights[L+1])) * sp;
+      // delta = DOT(delta, transpose(weight[L+1])) * sprime;
       for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
-        tmpdelta = 0;
+        float tmpdelta = 0.0;
         for (int i = 0; i < nnet[0].Nnodes[L+1]; i++) {
-          int k = j + i*nnet[0].Nnodes[L];
-          tmpdelta += nnet[0].weight[L][k] * nnet[0].delta[L+1][i]; // XXX check on the index values
+          int k = j + i*nnet[0].Nnodes[L]; // note order of (i,j) : j is [L+1] direction 
+          tmpdelta += nnet[0].weight[L][k] * nnet[0].delta[L+1][i];
         }
         nnet[0].delta[L][j] = tmpdelta * nnet[0].sprime[L][j];
@@ -179 +192 @@
     }                                          
 
-    // UPDATE
-    Nabla_b[L] = delta;
+    // NOTE on weight array: for a given layer, L, the matrix weight[L] maps the nodes in
+    // the previous layer (L-1) to those in L: Nnodes[L-1] -> Nnodes[L].  This is a matrix
+    // with dimensions (Nnodes[L-1] x Nnodes[L]).  The Nnodes[L-1] is the fast dimension,
+    // so for an element (i,j), the index k = i + j*Nnodes[L-1].   
+
+    // Nabla_b[L] = delta;
+    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
+      nnet[0]. Nabla_b[L][j] = nnet[0].delta[L][j];
+    }
     
-    // Nabla_w[L] = delta DOT transpose(nnet[0].svalue[L-1]);
+    // Nabla_w[L] = DOT(delta, transpose(svalue[L-1]));
     for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
       for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
-        int k = i + j*nnet[0].Nnodes[n-1];
-        nnet[0]. Nabla_w[n][k] = nnet[0].svalue[L-1][i] * nnet[0].delta[L][j];
-      }
-    }
-
+        int k = i + j*nnet[0].Nnodes[L-1];
+        nnet[0]. Nabla_w[L][k] = nnet[0].svalue[L-1][i] * nnet[0].delta[L][j];
+      }
+    }
   }
 }
 
 // support functions to loop over the Nabla entries
-void nnet_apply_Nabla (NNet *nnet, int Nmini, float eta) {
-  for (int n = 1; n < nnet[0].Nlayer; n++) {
-
-    for (int j = 0; j < nnet[0].Nnodes[n]; j++) {
-      nnet[0].biases[n][j] -= (eta / Nmini) * nnet[0].Nabla_b[n][j];
-
-      for (int i = 0; i < nnet[0].Nnodes[n-1]; i++) {
-        int k = i + j*nnet[0].Nnodes[n-1];
-        nnet[0].weights[n][k] -= (eta / Nmini) * nnet[0].Nabla_w[n][k];
-      }
-    }
-  }
-}
-void nnet_update_Nabla (NNet *nnet) {
-  for (int n = 1; n < nnet[0].Nlayer; n++) {
-
-    for (int j = 0; j < nnet[0].Nnodes[n]; j++) {
-      nnet[0]. Nabla_b[n][j] += nnet[0].dNabla_b[n][j];
-
-      for (int i = 0; i < nnet[0].Nnodes[n-1]; i++) {
-        int k = i + j*nnet[0].Nnodes[n-1];
-        nnet[0]. Nabla_w[n][k] += nnet[0].dNabla_w[n][k];
-      }
-    }
-  }
-}
-void nnet_reset_Nabla (NNet *nnet) {
-  for (int n = 1; n < nnet[0].Nlayer; n++) {
-
-    for (int j = 0; j < nnet[0].Nnodes[n]; j++) {
-
-      nnet[0]. Nabla_b[n][j] = 0;
-      nnet[0].dNabla_b[n][j] = 0;
-
-      for (int i = 0; i < nnet[0].Nnodes[n-1]; i++) {
-        int k = i + j*nnet[0].Nnodes[n-1];
-        nnet[0]. Nabla_w[n][k] = 0;
-        nnet[0].dNabla_w[n][k] = 0;
+void nnet_apply_Nabla (Nnet *nnet, int Nmini, float eta) {
+  for (int L = 1; L < nnet[0].Nlayer; L++) {
+
+    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
+      nnet[0].biases[L][j] -= (eta / Nmini) * nnet[0].Nabla_b[L][j];
+
+      for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
+        int k = i + j*nnet[0].Nnodes[L-1];
+        nnet[0].weight[L][k] -= (eta / Nmini) * nnet[0].Nabla_w[L][k];
+      }
+    }
+  }
+}
+void nnet_update_Nabla (Nnet *nnet) {
+  for (int L = 1; L < nnet[0].Nlayer; L++) {
+
+    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
+      nnet[0]. Nabla_b[L][j] += nnet[0].dNabla_b[L][j];
+
+      for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
+        int k = i + j*nnet[0].Nnodes[L-1];
+        nnet[0]. Nabla_w[L][k] += nnet[0].dNabla_w[L][k];
+      }
+    }
+  }
+}
+void nnet_reset_Nabla (Nnet *nnet) {
+  for (int L = 1; L < nnet[0].Nlayer; L++) {
+
+    for (int j = 0; j < nnet[0].Nnodes[L]; j++) {
+
+      nnet[0]. Nabla_b[L][j] = 0;
+      nnet[0].dNabla_b[L][j] = 0;
+
+      for (int i = 0; i < nnet[0].Nnodes[L-1]; i++) {
+        int k = i + j*nnet[0].Nnodes[L-1];
+        nnet[0]. Nabla_w[L][k] = 0;
+        nnet[0].dNabla_w[L][k] = 0;
       }
     }
@@ -238 +257 @@
 
 // the input values must already be copied to the input layer svalue[]
-void nnet_feedforward (NNet *nnet) {
+void nnet_feedforward (Nnet *nnet) {
 
   for (int i = 1; i < nnet[0].Nlayer; i++) {
@@ -246 +265 @@
 }
 
-// calcularte z, sigmoid(z) for each layer (z = w*value + bias)
-int nnet_onelayer (NNet *nnet, int n) {
-
-  if (n < 1) return FALSE; // abort here?
-  if (n >= nnet[0].Nlayer) return FALSE; // abort here?
-
-  // evaluating a single layer [n], n > 0, n < Nlayer:
-  int Ninput   = nnet[0].Nnodes[n - 1];
-  int Noutput  = nnet[0].Nnodes[n];
-
-  // input layer is [n-1], output layer is [n]
+// calculate z, sigmoid(z) for each layer (z = w*value + bias)
+int nnet_onelayer (Nnet *nnet, int L) {
+
+  if (L < 1) return FALSE; // abort here?
+  if (L >= nnet[0].Nlayer) return FALSE; // abort here?
+
+  // evaluating a single layer [L], n > 0, n < Nlayer:
+  int Ninput   = nnet[0].Nnodes[L - 1];
+  int Noutput  = nnet[0].Nnodes[L];
+
+  // input layer is [L-1], output layer is [L]
   for (int j = 0; j < Noutput; j ++) {
     float sum = 0;
@@ -262 +281 @@
       // weight matrix order is (0, 1, ... Ninput-1, Ninput, Ninput + 1, ... Ninput * Noutput - 1)
       int k = j * Ninput + i;
-      sum += nnet[0].weight[n][k]*nnet[0].svalue[n-1][i];
-    }
-    sum += nnet[0].biases[n][j];
-    nnet[0].zvalue[n][j] = sum;
-    nnet[0].svalue[n][j] = nnet_sigmoid(sum);
-    nnet[0].sprime[n][j] = nnet[0].svalue[n][j] * (1 - nnet[0].svalue[n][j]);
+      sum += nnet[0].weight[L][k]*nnet[0].svalue[L-1][i];
+    }
+    sum += nnet[0].biases[L][j];
+    nnet[0].zvalue[L][j] = sum;
+    nnet[0].svalue[L][j] = nnet_sigmoid(sum);
+    nnet[0].sprime[L][j] = nnet[0].svalue[L][j] * (1 - nnet[0].svalue[L][j]);
     // note that d sigmoid / dz = sigmoid * (1 - sigmoid)
   }
@@ -274 +293 @@
 
 float nnet_sigmoid (float value) {
-  return 1.0 / (1.0 + exp(-sum));
-}
-
-void sortseq (float *X, int *Y, int N) {
+  return 1.0 / (1.0 + exp(-value));
+}
+
+void nnet_sortseq (float *X, int *Y, int N) {
 
 # define SWAPFUNC(A,B){ float ftmp; int itmp; \