log individual loss types and average losses

justadudewhohacks · justadudewhohacks · commit 0320da07bc91 · 2018-08-10T09:29:08.000+02:00
diff --git a/tools/train/tinyYolov2/overfit.html b/tools/train/tinyYolov2/overfit.html
@@ -17,25 +17,26 @@
   <script>
     tf = faceapi.tf
 
-    const weightsUrl = '/tmp/test.weights'
-    const fromEpoch = 800
+    const weightsUrl = '/tmp/initial_tiny_yolov2_glorot_normal.weights'
+    const fromEpoch = 0
 
 
-    window.debug = true
-    window.logTrainSteps = true
+    window.debug = false
+    window.logTrainSteps = false
 
 
     // hyper parameters
     window.objectScale = 5
     window.noObjectScale = 1
     window.coordScale = 1
 
-    window.saveEveryNthIteration = 50
+    window.saveEveryNthIteration = 1
     window.trainSteps = 4000
     //window.optimizer = tf.train.sgd(0.001)
     window.optimizer = tf.train.adam(0.001, 0.9, 0.999, 1e-8)
 
-    const numTrainSamples = 1
+    // all samples
+    const numTrainSamples = Infinity
 
     async function loadNetWeights(uri) {
       return new Float32Array(await (await fetch(uri)).arrayBuffer())
@@ -50,13 +51,18 @@
       window.net = new faceapi.TinyYolov2(true)
       window.net.load(weights)
       window.net.variable()
-      window.detectionFilenames = (await fetchDetectionFilenames()).slice(1, numTrainSamples + 1)
+      window.detectionFilenames = (await fetchDetectionFilenames()).slice(0, numTrainSamples)
       window.lossMap = {}
 
       console.log('ready')
     }
 
-    const trainSizes = [608]
+    const trainSizes = [320]
+
+    function logLossChange(lossType) {
+      const { currentLoss, prevLoss, detectionFilenames } = window
+      log(`${lossType} : ${faceapi.round(currentLoss[lossType])} (avg: ${faceapi.round(currentLoss[lossType] / detectionFilenames.length)}) (delta: ${currentLoss[lossType] - prevLoss[lossType]})`)
+    }
 
     async function train(batchSize = 1) {
       for (let i = fromEpoch; i < trainSteps; i++) {
@@ -81,21 +87,30 @@
         log()
         log('step %s done (%s ms)', i, ts)
 
-        const currentLoss = Object.keys(lossMap).map(k => lossMap[k]).reduce((sum, l) => sum + l)
+        window.prevLoss = window.currentLoss
+        window.currentLoss = Object.keys(lossMap)
+          .map(filename => lossMap[filename])
+          .reduce((accumulatedLosses, losses) =>
+            Object.keys(losses)
+              .map(key => ({
+                [key]: (accumulatedLosses[key] || 0) + losses[key]
+              }))
+              .reduce((map, curr) => ({ ...map, ...curr }), {}),
+            {}
+          )
+
         if (window.prevLoss) {
-          log('prevLoss:', window.prevLoss)
-          log('currentLoss:', currentLoss)
-          log('loss change:', currentLoss - window.prevLoss)
+          logLossChange('noObjectLoss')
+          logLossChange('objectLoss')
+          logLossChange('coordLoss')
+          logLossChange('totalLoss')
         }
         log()
         log('--------------------')
         log()
 
-        window.prevLoss = currentLoss
-
-
         if (((i + 1) % saveEveryNthIteration) === 0) {
-          saveWeights(window.net, 'adam_511_' + i + '.weights')
+          saveWeights(window.net, 'adam_511_' + (i + 1) + '.weights')
         }
       }
     }
diff --git a/tools/train/tinyYolov2/train.js b/tools/train/tinyYolov2/train.js
@@ -25,22 +25,26 @@ async function trainStep(batchCreators, inputSize) {
         batchInput.getRelativePaddings(batchIdx)
       )
 
-
-      const total = totalLoss.dataSync()[0]
+      const losses = {
+        totalLoss: totalLoss.dataSync()[0],
+        noObjectLoss: noObjectLoss.dataSync()[0],
+        objectLoss: objectLoss.dataSync()[0],
+        coordLoss: coordLoss.dataSync()[0]
+      }
 
       if (window.logTrainSteps) {
         log(`ground truth boxes: ${groundTruthBoxes[batchIdx].length}`)
-        log(`noObjectLoss[${dataIdx}]: ${noObjectLoss.dataSync()}`)
-        log(`objectLoss[${dataIdx}]: ${objectLoss.dataSync()}`)
-        log(`coordLoss[${dataIdx}]: ${coordLoss.dataSync()}`)
-        log(`totalLoss[${dataIdx}]: ${total}`)
+        log(`noObjectLoss[${dataIdx}]: ${losses.noObjectLoss}`)
+        log(`objectLoss[${dataIdx}]: ${losses.objectLoss}`)
+        log(`coordLoss[${dataIdx}]: ${losses.coordLoss}`)
+        log(`totalLoss[${dataIdx}]: ${losses.totalLoss}`)
 
         if (window.lossMap[filenames]) {
-          log(`loss change: ${total - window.lossMap[filenames]}`)
+          log(`loss change: ${losses.totalLoss - window.lossMap[filenames].totalLoss}`)
         }
       }
 
-      window.lossMap[filenames] = total
+      window.lossMap[filenames] = losses
 
       return totalLoss
     }, true)
diff --git a/tools/train/tinyYolov2/verify.html b/tools/train/tinyYolov2/verify.html
@@ -138,7 +138,7 @@
 
     async function run() {
       $('#imgByNr').keydown(onKeyDown)
-      const weights = await loadNetWeights('/tmp/test2.weights')
+      const weights = await loadNetWeights('/tmp/test_n100_320_114.weights')
       window.net = new faceapi.TinyYolov2(true)
       await window.net.load(weights)
 
