Skip to content

Commit e8373d5

Browse files
justadudewhohacksjustadudewhohacks
committed
implemented withFaceExpressions
1 parent da8b102 commit e8373d5

File tree

9 files changed

+110
-78
lines changed

9 files changed

+110
-78
lines changed

‎src/faceExpressionNet/FaceExpressionNet.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { NetInput, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
44
import { FaceFeatureExtractor } from '../faceFeatureExtractor/FaceFeatureExtractor';
55
import { FaceFeatureExtractorParams } from '../faceFeatureExtractor/types';
66
import { FaceProcessor } from '../faceProcessor/FaceProcessor';
7-
import { faceExpressionLabels } from './types';
7+
import { FaceExpression, faceExpressionLabels, FaceExpressionPrediction } from './types';
88

99
export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> {
1010

@@ -18,12 +18,12 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams>
1818
return label
1919
}
2020

21-
public static decodeProbabilites(probabilities: number[] | Float32Array) {
21+
public static decodeProbabilites(probabilities: number[] | Float32Array): FaceExpressionPrediction[] {
2222
if (probabilities.length !== 7) {
2323
throw new Error(`decodeProbabilites - expected probabilities.length to be 7, have: ${probabilities.length}`)
2424
}
2525

26-
return Object.keys(faceExpressionLabels)
26+
return (Object.keys(faceExpressionLabels) as FaceExpression[])
2727
.map(expression => ({ expression, probability: probabilities[faceExpressionLabels[expression]] }))
2828
}
2929

‎src/faceExpressionNet/types.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,11 @@ export const faceExpressionLabels = {
66
fearful: 4,
77
disgusted: 5,
88
surprised:6
9-
}
9+
}
10+
11+
export type FaceExpression = 'neutral' | 'happy' | 'sad' | 'angry' | 'fearful' | 'disgusted' | 'surprised'
12+
13+
export type FaceExpressionPrediction = {
14+
expression: FaceExpression,
15+
probability: number
16+
}

‎src/faceFeatureExtractor/FaceFeatureExtractor.ts

Lines changed: 6 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,10 @@
11
import * as tf from '@tensorflow/tfjs-core';
22
import { NetInput, NeuralNetwork, normalize, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
3-
import { ConvParams, SeparableConvParams } from 'tfjs-tiny-yolov2';
43

5-
import { depthwiseSeparableConv } from './depthwiseSeparableConv';
4+
import { denseBlock4 } from './denseBlock';
65
import { extractParams } from './extractParams';
76
import { extractParamsFromWeigthMap } from './extractParamsFromWeigthMap';
8-
import { DenseBlock4Params, FaceFeatureExtractorParams, IFaceFeatureExtractor } from './types';
9-
10-
function denseBlock(
11-
x: tf.Tensor4D,
12-
denseBlockParams: DenseBlock4Params,
13-
isFirstLayer: boolean = false
14-
): tf.Tensor4D {
15-
return tf.tidy(() => {
16-
const out1 = tf.relu(
17-
isFirstLayer
18-
? tf.add(
19-
tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, [2, 2], 'same'),
20-
denseBlockParams.conv0.bias
21-
)
22-
: depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, [2, 2])
23-
) as tf.Tensor4D
24-
const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
25-
26-
const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
27-
const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
28-
29-
const in4 = tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
30-
const out4 = depthwiseSeparableConv(in4, denseBlockParams.conv3, [1, 1])
31-
32-
return tf.relu(tf.add(out1, tf.add(out2, tf.add(out3, out4)))) as tf.Tensor4D
33-
})
34-
}
7+
import { FaceFeatureExtractorParams, IFaceFeatureExtractor } from './types';
358

369
export class FaceFeatureExtractor extends NeuralNetwork<FaceFeatureExtractorParams> implements IFaceFeatureExtractor<FaceFeatureExtractorParams> {
3710

@@ -52,10 +25,10 @@ export class FaceFeatureExtractor extends NeuralNetwork<FaceFeatureExtractorPara
5225
const meanRgb = [122.782, 117.001, 104.298]
5326
const normalized = normalize(batchTensor, meanRgb).div(tf.scalar(255)) as tf.Tensor4D
5427

55-
let out = denseBlock(normalized, params.dense0, true)
56-
out = denseBlock(out, params.dense1)
57-
out = denseBlock(out, params.dense2)
58-
out = denseBlock(out, params.dense3)
28+
let out = denseBlock4(normalized, params.dense0, true)
29+
out = denseBlock4(out, params.dense1)
30+
out = denseBlock4(out, params.dense2)
31+
out = denseBlock4(out, params.dense3)
5932
out = tf.avgPool(out, [7, 7], [2, 2], 'valid')
6033

6134
return out

‎src/faceFeatureExtractor/TinyFaceFeatureExtractor.ts

Lines changed: 6 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,10 @@
11
import * as tf from '@tensorflow/tfjs-core';
22
import { NetInput, NeuralNetwork, normalize, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
3-
import { ConvParams, SeparableConvParams } from 'tfjs-tiny-yolov2';
43

5-
import { depthwiseSeparableConv } from './depthwiseSeparableConv';
4+
import { denseBlock3 } from './denseBlock';
65
import { extractParamsFromWeigthMapTiny } from './extractParamsFromWeigthMapTiny';
76
import { extractParamsTiny } from './extractParamsTiny';
8-
import { DenseBlock3Params, IFaceFeatureExtractor, TinyFaceFeatureExtractorParams } from './types';
9-
10-
function denseBlock(
11-
x: tf.Tensor4D,
12-
denseBlockParams: DenseBlock3Params,
13-
isFirstLayer: boolean = false
14-
): tf.Tensor4D {
15-
return tf.tidy(() => {
16-
const out1 = tf.relu(
17-
isFirstLayer
18-
? tf.add(
19-
tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, [2, 2], 'same'),
20-
denseBlockParams.conv0.bias
21-
)
22-
: depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, [2, 2])
23-
) as tf.Tensor4D
24-
const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
25-
26-
const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
27-
const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
28-
29-
return tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
30-
})
31-
}
7+
import { IFaceFeatureExtractor, TinyFaceFeatureExtractorParams } from './types';
328

339
export class TinyFaceFeatureExtractor extends NeuralNetwork<TinyFaceFeatureExtractorParams> implements IFaceFeatureExtractor<TinyFaceFeatureExtractorParams> {
3410

@@ -49,9 +25,9 @@ export class TinyFaceFeatureExtractor extends NeuralNetwork<TinyFaceFeatureExtra
4925
const meanRgb = [122.782, 117.001, 104.298]
5026
const normalized = normalize(batchTensor, meanRgb).div(tf.scalar(255)) as tf.Tensor4D
5127

52-
let out = denseBlock(normalized, params.dense0, true)
53-
out = denseBlock(out, params.dense1)
54-
out = denseBlock(out, params.dense2)
28+
let out = denseBlock3(normalized, params.dense0, true)
29+
out = denseBlock3(out, params.dense1)
30+
out = denseBlock3(out, params.dense2)
5531
out = tf.avgPool(out, [14, 14], [2, 2], 'valid')
5632

5733
return out
@@ -63,7 +39,7 @@ export class TinyFaceFeatureExtractor extends NeuralNetwork<TinyFaceFeatureExtra
6339
}
6440

6541
protected getDefaultModelName(): string {
66-
return 'face_landmark_68_tiny_model'
42+
return 'face_feature_extractor_tiny_model'
6743
}
6844

6945
protected extractParamsFromWeigthMap(weightMap: tf.NamedTensorMap) {

‎src/faceFeatureExtractor/denseBlock.ts

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import * as tf from '@tensorflow/tfjs-core';
2+
import { ConvParams, SeparableConvParams } from 'tfjs-tiny-yolov2';
3+
4+
import { depthwiseSeparableConv } from './depthwiseSeparableConv';
5+
import { DenseBlock3Params, DenseBlock4Params } from './types';
6+
7+
export function denseBlock3(
8+
x: tf.Tensor4D,
9+
denseBlockParams: DenseBlock3Params,
10+
isFirstLayer: boolean = false
11+
): tf.Tensor4D {
12+
return tf.tidy(() => {
13+
const out1 = tf.relu(
14+
isFirstLayer
15+
? tf.add(
16+
tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, [2, 2], 'same'),
17+
denseBlockParams.conv0.bias
18+
)
19+
: depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, [2, 2])
20+
) as tf.Tensor4D
21+
const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
22+
23+
const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
24+
const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
25+
26+
return tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
27+
})
28+
}
29+
30+
export function denseBlock4(
31+
x: tf.Tensor4D,
32+
denseBlockParams: DenseBlock4Params,
33+
isFirstLayer: boolean = false,
34+
isScaleDown: boolean = true
35+
): tf.Tensor4D {
36+
return tf.tidy(() => {
37+
const out1 = tf.relu(
38+
isFirstLayer
39+
? tf.add(
40+
tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, isScaleDown ? [2, 2] : [1, 1], 'same'),
41+
denseBlockParams.conv0.bias
42+
)
43+
: depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, isScaleDown ? [2, 2] : [1, 1])
44+
) as tf.Tensor4D
45+
const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
46+
47+
const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
48+
const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
49+
50+
const in4 = tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
51+
const out4 = depthwiseSeparableConv(in4, denseBlockParams.conv3, [1, 1])
52+
53+
return tf.relu(tf.add(out1, tf.add(out2, tf.add(out3, out4)))) as tf.Tensor4D
54+
})
55+
}

‎src/factories/WithFaceExpressions.ts

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
1-
// TODO
2-
export type FaceExpressions = number[]
1+
import { FaceExpressionPrediction } from '../faceExpressionNet/types';
32

43
export type WithFaceExpressions<TSource> = TSource & {
5-
expressions: FaceExpressions
4+
expressions: FaceExpressionPrediction[]
65
}
76

87
export function extendWithFaceExpressions<
98
TSource
109
> (
1110
sourceObj: TSource,
12-
expressions: FaceExpressions
11+
expressions: FaceExpressionPrediction[]
1312
): WithFaceExpressions<TSource> {
1413

1514
const extension = { expressions }

‎src/globalApi/DetectFaceLandmarksTasks.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ export class DetectSingleFaceLandmarksTask<
7777
? await extractFaceTensors(this.input, [detection])
7878
: await extractFaces(this.input, [detection])
7979

80-
8180
const landmarks = await this.landmarkNet.detectLandmarks(faces[0]) as FaceLandmarks68
8281

8382
faces.forEach(f => f instanceof tf.Tensor && f.dispose())

‎src/globalApi/PredictFaceExpressionsTask.ts

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
import { TNetInput } from 'tfjs-image-recognition-base';
2+
import { tf } from 'tfjs-tiny-yolov2';
23

4+
import { extractFaces, extractFaceTensors } from '../dom';
5+
import { FaceExpressionPrediction } from '../faceExpressionNet/types';
36
import { WithFaceDetection } from '../factories/WithFaceDetection';
47
import { extendWithFaceExpressions, WithFaceExpressions } from '../factories/WithFaceExpressions';
58
import { ComposableTask } from './ComposableTask';
69
import { DetectAllFaceLandmarksTask, DetectSingleFaceLandmarksTask } from './DetectFaceLandmarksTasks';
10+
import { nets } from './nets';
711

812
export class PredictFaceExpressionsTaskBase<TReturn, TParentReturn> extends ComposableTask<TReturn> {
913
constructor(
@@ -22,9 +26,20 @@ export class PredictAllFaceExpressionsTask<
2226

2327
const parentResults = await this.parentTask
2428

25-
// TODO: implement me
29+
const detections = parentResults.map(parentResult => parentResult.detection)
30+
const faces: Array<HTMLCanvasElement | tf.Tensor3D> = this.input instanceof tf.Tensor
31+
? await extractFaceTensors(this.input, detections)
32+
: await extractFaces(this.input, detections)
2633

27-
return parentResults.map(parentResult => extendWithFaceExpressions<TSource>(parentResult, []))
34+
const faceExpressionsByFace = await Promise.all(faces.map(
35+
face => nets.faceExpressionNet.predictExpressions(face)
36+
)) as FaceExpressionPrediction[][]
37+
38+
faces.forEach(f => f instanceof tf.Tensor && f.dispose())
39+
40+
return parentResults.map(
41+
(parentResult, i) => extendWithFaceExpressions<TSource>(parentResult, faceExpressionsByFace[i])
42+
)
2843
}
2944

3045
withFaceLandmarks(): DetectAllFaceLandmarksTask<WithFaceExpressions<TSource>> {
@@ -43,9 +58,16 @@ export class PredictSingleFaceExpressionTask<
4358
return
4459
}
4560

46-
// TODO: implement me
61+
const { detection } = parentResult
62+
const faces: Array<HTMLCanvasElement | tf.Tensor3D> = this.input instanceof tf.Tensor
63+
? await extractFaceTensors(this.input, [detection])
64+
: await extractFaces(this.input, [detection])
65+
66+
const faceExpressions = await nets.faceExpressionNet.predictExpressions(faces[0]) as FaceExpressionPrediction[]
67+
68+
faces.forEach(f => f instanceof tf.Tensor && f.dispose())
4769

48-
return extendWithFaceExpressions(parentResult, [])
70+
return extendWithFaceExpressions(parentResult, faceExpressions)
4971
}
5072

5173
withFaceLandmarks(): DetectSingleFaceLandmarksTask<WithFaceExpressions<TSource>> {

‎src/globalApi/nets.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { ITinyYolov2Options } from 'tfjs-tiny-yolov2';
44
import { FaceDetection } from '../classes/FaceDetection';
55
import { FaceLandmarks5 } from '../classes/FaceLandmarks5';
66
import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
7-
import { FaceFeatureExtractor, TinyFaceFeatureExtractor } from '../faceFeatureExtractor';
7+
import { FaceExpressionNet } from '../faceExpressionNet/FaceExpressionNet';
88
import { FaceLandmark68Net } from '../faceLandmarkNet/FaceLandmark68Net';
99
import { FaceLandmark68TinyNet } from '../faceLandmarkNet/FaceLandmark68TinyNet';
1010
import { FaceRecognitionNet } from '../faceRecognitionNet/FaceRecognitionNet';
@@ -25,7 +25,8 @@ export const nets = {
2525
mtcnn: new Mtcnn(),
2626
faceLandmark68Net: new FaceLandmark68Net(),
2727
faceLandmark68TinyNet: new FaceLandmark68TinyNet(),
28-
faceRecognitionNet: new FaceRecognitionNet()
28+
faceRecognitionNet: new FaceRecognitionNet(),
29+
faceExpressionNet: new FaceExpressionNet()
2930
}
3031

3132
/**

0 commit comments

Comments
 (0)