Skip to content

Commit ebaf7f2

Browse files
justadudewhohacksjustadudewhohacks
committed
fixed train and testdata splitting + init test script
1 parent 883f20c commit ebaf7f2

File tree

9 files changed

+285
-26
lines changed

9 files changed

+285
-26
lines changed

‎src/faceExpressionNet/FaceExpressionNet.ts

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@ import { NetInput, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
44
import { FaceFeatureExtractor } from '../faceFeatureExtractor/FaceFeatureExtractor';
55
import { FaceFeatureExtractorParams } from '../faceFeatureExtractor/types';
66
import { FaceProcessor } from '../faceProcessor/FaceProcessor';
7-
import { EmotionLabels } from './types';
7+
import { emotionLabels } from './types';
88

99
export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> {
1010

1111
public static getEmotionLabel(emotion: string) {
12-
const label = EmotionLabels[emotion.toUpperCase()]
12+
const label = emotionLabels[emotion]
1313

1414
if (typeof label !== 'number') {
1515
throw new Error(`getEmotionLabel - no label for emotion: ${emotion}`)
@@ -22,7 +22,8 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams>
2222
if (probabilities.length !== 7) {
2323
throw new Error(`decodeEmotions - expected probabilities.length to be 7, have: ${probabilities.length}`)
2424
}
25-
return Object.keys(EmotionLabels).map(label => ({ label, probability: probabilities[EmotionLabels[label]] }))
25+
26+
return Object.keys(emotionLabels).map(label => ({ label, probability: probabilities[emotionLabels[label]] }))
2627
}
2728

2829
constructor(faceFeatureExtractor: FaceFeatureExtractor = new FaceFeatureExtractor()) {
@@ -45,11 +46,24 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams>
4546
}
4647

4748
public async predictExpressions(input: TNetInput) {
48-
const out = await this.forward(input)
49+
const netInput = await toNetInput(input)
50+
const out = await this.forwardInput(netInput)
4951
const probabilitesByBatch = await Promise.all(tf.unstack(out).map(t => t.data()))
5052
out.dispose()
5153

52-
return probabilitesByBatch.map(propablities => FaceExpressionNet.decodeEmotions(propablities as Float32Array))
54+
const predictionsByBatch = probabilitesByBatch
55+
.map(propablities => {
56+
const predictions = {}
57+
FaceExpressionNet.decodeEmotions(propablities as Float32Array)
58+
.forEach(({ label, probability }) => {
59+
predictions[label] = probability
60+
})
61+
return predictions
62+
})
63+
64+
return netInput.isBatchInput
65+
? predictionsByBatch
66+
: predictionsByBatch[0]
5367
}
5468

5569
public dispose(throwOnRedispose: boolean = true) {

‎src/faceExpressionNet/types.ts

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
export enum EmotionLabels {
2-
NEUTRAL = 0,
3-
HAPPY = 1,
4-
SAD = 2,
5-
ANGRY = 3,
6-
FEARFUL = 4,
7-
DISGUSTED = 5,
8-
SURPRISED = 6
1+
export const emotionLabels = {
2+
neutral: 0,
3+
happy: 1,
4+
sad: 2,
5+
angry: 3,
6+
fearful: 4,
7+
disgusted: 5,
8+
surprised:6
99
}

‎tools/train/faceExpressions/public/testClassifier.html

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
<script src="face-api.js"></script>
5+
<script src="FileSaver.js"></script>
6+
<script src="js/commons.js"></script>
7+
</head>
8+
<body>
9+
<div id="container"></div>
10+
11+
<script>
12+
tf = faceapi.tf
13+
14+
// load the FaceLandmark68Net and use it's feature extractor since we only
15+
// train the output layer of the FaceExpressionNet
16+
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
17+
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
18+
19+
// uri to weights file of last checkpoint
20+
const modelCheckpoint = 'tmp/face_expression_model_165.weights'
21+
22+
async function load() {
23+
window.testData = await faceapi.fetchJson('testData.json')
24+
await dummyLandmarkNet.load('/')
25+
26+
// fetch the actual output layer weights
27+
const classifierWeights = await faceapi.fetchNetWeights(modelCheckpoint)
28+
await window.net.loadClassifierParams(classifierWeights)
29+
30+
console.log('loaded')
31+
}
32+
33+
load()
34+
35+
async function test() {
36+
const emotions = Object.keys(window.testData)
37+
let errors = {}
38+
let preds = {}
39+
let sizes = {}
40+
41+
for (let emotion of emotions) {
42+
43+
const container = document.getElementById('container')
44+
const span = document.createElement('div')
45+
container.appendChild(span)
46+
47+
console.log(emotion)
48+
49+
const dataForLabel = window.testData[emotion]
50+
51+
errors[emotion] = 0
52+
preds[emotion] = 0
53+
sizes[emotion] = dataForLabel.length
54+
55+
56+
for (let [idx, data] of dataForLabel.entries()) {
57+
span.innerHTML = emotion + ': ' + faceapi.round(idx / dataForLabel.length) * 100 + '%'
58+
59+
const img = await faceapi.fetchImage(getImageUrl({ ...data, label: emotion }))
60+
const pred = await window.net.predictExpressions(img)
61+
const bestPred = Object.keys(pred)
62+
.map(label => ({ label, probability: pred[label] }))
63+
.reduce((best, curr) => curr.probability < best.probability ? curr : best)
64+
errors[emotion] += (1 - pred[emotion])
65+
pred[emotion] += (bestPred.label === emotion ? 1 : 0)
66+
67+
}
68+
69+
span.innerHTML = emotion + ': 100%'
70+
71+
}
72+
73+
const totalError = emotions.reduce((err, emotion) => err + errors[emotion], 0)
74+
75+
console.log('done...')
76+
console.log('test set size:', sizes)
77+
console.log('preds:', preds)
78+
console.log('errors:', errors)
79+
console.log('total error:', totalError)
80+
}
81+
82+
</script>
83+
</body>
84+
</html>

‎tools/train/faceExpressions/public/testData.json

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎tools/train/faceExpressions/public/trainClassifier.html

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,15 @@
1717
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
1818

1919
// uri to weights file of last checkpoint
20-
const modelCheckpoint = 'tmp/initial_classifier.weights'
21-
const startEpoch = 0
20+
const modelCheckpoint = 'tmp/face_expression_model_148.weights'
21+
const startEpoch = 149
2222

23-
const learningRate = 0.001 // 0.001
23+
const learningRate = 0.001
2424
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
2525

2626
window.saveEveryNthSample = Infinity
2727

28-
window.batchSize = 16
28+
window.batchSize = 32
2929
//window.batchSize = 32
3030

3131
window.lossValues = []
@@ -57,7 +57,7 @@
5757
saveWeights(window.net, `face_expression_model_${epoch}.weights`)
5858

5959
const loss = window.lossValues[epoch]
60-
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / window.trainIds.length })]), `face_expression_model_${epoch}.json`)
60+
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / (2000 * 7) })]), `face_expression_model_${epoch}.json`)
6161

6262
}
6363

‎tools/train/faceExpressions/public/trainClassifier_debug.html

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
<script src="face-api.js"></script>
5+
<script src="FileSaver.js"></script>
6+
<script src="js/commons.js"></script>
7+
</head>
8+
<body>
9+
<div id="container"></div>
10+
<div id="template" style="display: inline-flex; flex-direction: column;">
11+
<span class="emotion-text"></span>
12+
<span class="predicted-text"></span>
13+
</div>
14+
15+
<script>
16+
tf = faceapi.tf
17+
18+
// load the FaceLandmark68Net and use it's feature extractor since we only
19+
// train the output layer of the FaceExpressionNet
20+
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
21+
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
22+
23+
// uri to weights file of last checkpoint
24+
const modelCheckpoint = 'tmp/initial_classifier.weights'
25+
const startEpoch = 0
26+
27+
const learningRate = 0.1 // 0.001
28+
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
29+
30+
window.batchSize = 32
31+
32+
window.iterDelay = 0
33+
window.withLogging = true
34+
35+
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
36+
37+
async function load() {
38+
window.trainData = await faceapi.fetchJson('trainData.json')
39+
await dummyLandmarkNet.load('/')
40+
41+
// fetch the actual output layer weights
42+
const classifierWeights = await faceapi.fetchNetWeights(modelCheckpoint)
43+
await window.net.loadClassifierParams(classifierWeights)
44+
window.net.variable()
45+
}
46+
47+
function prepareDataForEpoch() {
48+
return faceapi.shuffleArray(
49+
Object.keys(window.trainData).map(label => {
50+
let dataForLabel = window.trainData[label].map(data => ({ ...data, label }))
51+
// since train data for "disgusted" have less than 2000 samples
52+
// use some data twice to ensure an even distribution
53+
dataForLabel = label === 'disgusted'
54+
? faceapi.shuffleArray(dataForLabel.concat(dataForLabel).concat(dataForLabel)).slice(0, 2000)
55+
: dataForLabel
56+
return dataForLabel
57+
}).reduce((flat, arr) => arr.concat(flat))
58+
)
59+
}
60+
61+
function getLabelOneHotVector(emotion) {
62+
const label = faceapi.FaceExpressionNet.getEmotionLabel(emotion)
63+
return Array(7).fill(0).map((_, i) => i === label ? 1 : 0)
64+
}
65+
66+
async function train() {
67+
await load()
68+
69+
const shuffledInputs = prepareDataForEpoch().slice(0, window.batchSize)
70+
const batchData = shuffledInputs
71+
const bImages = await Promise.all(
72+
batchData
73+
.map(data => getImageUrl(data))
74+
.map(imgUrl => faceapi.fetchImage(imgUrl))
75+
)
76+
const bOneHotVectors = batchData
77+
.map(data => getLabelOneHotVector(data.label))
78+
79+
const container = document.getElementById('container')
80+
const template = document.getElementById('template')
81+
82+
bImages.forEach((img, i) => {
83+
console.log(i, batchData[i].label, batchData[i].img)
84+
85+
const squaredImg = faceapi.imageToSquare(img, 112, true)
86+
const emotions = faceapi.FaceExpressionNet
87+
.decodeEmotions(bOneHotVectors[i])
88+
.filter(e => e.probability > 0)
89+
90+
const clone = template.cloneNode(true)
91+
clone.id = i
92+
const span = clone.firstElementChild
93+
span.innerHTML = i + ':' + emotions[0].label
94+
clone.insertBefore(squaredImg, span)
95+
container.appendChild(clone)
96+
})
97+
98+
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
99+
100+
const bottleneckFeatures = await window.net.faceFeatureExtractor.forward(bImages)
101+
102+
const loss = optimizer.minimize(() => {
103+
const labels = tf.tensor2d(bOneHotVectors)
104+
const out = window.net.forwardInput(bottleneckFeatures)
105+
106+
const loss = tf.losses.softmaxCrossEntropy(
107+
labels,
108+
out,
109+
tf.Reduction.MEAN
110+
)
111+
112+
const predictedByBatch = tf.unstack(out)
113+
predictedByBatch.forEach((p, i) => {
114+
const probabilities = Array.from(p.dataSync())
115+
const emotions = faceapi.FaceExpressionNet.decodeEmotions(probabilities)
116+
const container = document.getElementById(i)
117+
118+
const pred = emotions.reduce((best, curr) => curr.probability > best.probability ? curr : best)
119+
120+
const predNode = container.children[container.children.length - 1]
121+
122+
predNode.innerHTML =
123+
pred.label + ' (' + faceapi.round(pred.probability) + ')'
124+
})
125+
126+
return loss
127+
}, true)
128+
129+
bottleneckFeatures.dispose()
130+
131+
// start next iteration without waiting for loss data
132+
133+
loss.data().then(data => {
134+
const lossValue = data[0]
135+
log(`epoch ${epoch}, loss: ${lossValue}`)
136+
loss.dispose()
137+
})
138+
139+
if (window.iterDelay) {
140+
await delay(window.iterDelay)
141+
} else {
142+
await tf.nextFrame()
143+
}
144+
}
145+
}
146+
147+
</script>
148+
</body>
149+
</html>

‎tools/train/faceExpressions/public/trainData.json

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎tools/train/faceExpressions/server.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,6 @@ app.use(express.static(path.resolve(process.env.DATA_PATH)))
1616

1717
app.get('/', (req, res) => res.redirect('/train'))
1818
app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'trainClassifier.html')))
19+
app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'testClassifier.html')))
1920

2021
app.listen(8000, () => console.log('Listening on port 8000!'))

‎tools/train/faceExpressions/splitDataSet.js

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,14 @@ const MAX_TRAIN_SAMPLES_PER_CLASS = 2000
3434
require('./.env')
3535
const { shuffleArray } = require('../../../')
3636
const fs = require('fs')
37+
const path = require('path')
3738

38-
const createImageNameArray = (db, num, ext) =>
39-
Array(num).fill(0)
40-
.map((_, i) => `${i}${ext}`)
41-
.map(img => ({ db, img }))
39+
const dbEmotionMapping = JSON.parse(fs.readFileSync(
40+
path.resolve(
41+
process.env.DATA_PATH,
42+
'face-expressions/emotionMapping.json'
43+
)
44+
).toString())
4245

4346
const splitArray = (arr, idx) => [arr.slice(0, idx), arr.slice(idx)]
4447

@@ -53,8 +56,16 @@ Object.keys(dataDistribution)
5356
const numDb = Math.floor(Math.min(0.7 * MAX_TRAIN_SAMPLES_PER_CLASS, 0.7 * db))
5457
const numKaggle = Math.floor(Math.min(MAX_TRAIN_SAMPLES_PER_CLASS - numDb, 0.7 * kaggle))
5558

56-
const dbImages = shuffleArray(createImageNameArray('db', db, '.jpg'))
57-
const kaggleImages = shuffleArray(createImageNameArray('kaggle', kaggle, '.png'))
59+
const dbImages = shuffleArray(
60+
dbEmotionMapping[label]
61+
.map(img => ({ db: 'db', img }))
62+
)
63+
const kaggleImages = shuffleArray(
64+
Array(kaggle).fill(0).map((_, i) => `${i}.png`)
65+
.map(img => ({ db: 'kaggle', img }))
66+
)
67+
68+
5869

5970
const [dbTrain, dbTest] = splitArray(dbImages, numDb)
6071
const [kaggleTrain, kaggleTest] = splitArray(kaggleImages, numKaggle)

0 commit comments

Comments
 (0)