yoloserv/modules/paravision/recognition/tests/test_sdk_enhanced.py

496 lines
20 KiB
Python

import os
import numpy as np
from unittest import TestCase
from ..sdk import SDK
from ..types import ImageInferenceData
from ..engine import Engine
from ..utils import load_image
from ..types import BoundingBox, ScoringMode, Embedding
from ..exceptions import InvalidInputException
from .utils import is_json_serializable
from .constants import (
IMG_NOFACE,
IMG_ONEFACE,
IMG_MANYFACES,
IMG_IDENTITY1_FACE1,
IMG_IDENTITY1_FACE2,
IMG_ONEFACE_RECO_INPUT_IMG,
ERR_MISSING_BBOX,
ERR_MISSING_SCORE,
ERR_MISSING_LANDMARKS,
ERR_MISSING_EMBEDDING,
ERR_MISSING_MASK_PROB,
ERR_MISSING_FACES,
ERR_JSON_FACE,
ERR_UNEXPECTED_LANDMARKS,
ERR_UNEXPECTED_QUALITY,
ERR_UNEXPECTED_NUM_FACES,
ERR_UNEXPECTED_NUM_INFERENCES,
ERR_UNEXPECTED_AGES,
ERR_UNEXPECTED_GENDERS,
ERR_UNEXPECTED_AGE,
ERR_UNEXPECTED_GENDER,
ERR_INVALID_MASK_PROB,
ERR_INVALID_MPF,
MAX_NO_MASK_SCORE,
MASK_SCORE,
ERR_INVALID_SCORING_MODE,
ERR_INVALID_EMBEDDING_SIZE,
ERR_INVALID_AGES,
EXPECTED_ENHANCED_EMBED_LEN,
)
ASSETS_PATH = os.path.join(os.path.dirname(__file__), "assets")
engine_default = None
scoring_mode = None
sdk = None
class TestSDK(TestCase):
@classmethod
def setUpClass(cls):
global sdk
global engine_default
global scoring_mode
engine_default = Engine.OPENVINO
scoring_mode = ScoringMode.EnhancedEmbedding
sdk = SDK(engine=engine_default, settings={"scoring_mode": scoring_mode})
def setUp(self):
self.sdk = sdk
def test_load_image_invalid_input(self):
with self.assertRaises(InvalidInputException):
load_image("invalid-img.jpg")
def test_empty_case(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_NOFACE))]
detection_result = self.sdk.get_faces(
imgs, qualities=True, landmarks=True, embeddings=True
)
faces = detection_result.faces
self.assertEqual(len(faces), 0, msg=ERR_UNEXPECTED_NUM_FACES)
image_inferences = detection_result.image_inferences
self.assertEqual(len(image_inferences), 1, msg=ERR_UNEXPECTED_NUM_INFERENCES)
detection_result = self.sdk.get_bounding_boxes(imgs)
self.assertEqual(len(detection_result.faces), 0, msg=ERR_UNEXPECTED_NUM_FACES)
def test_get_faces(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
detection_result = self.sdk.get_faces(
imgs, qualities=True, landmarks=True, embeddings=True
)
faces = detection_result.faces
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
image_inferences = detection_result.image_inferences
self.assertEqual(len(image_inferences), 1, msg=ERR_UNEXPECTED_NUM_INFERENCES)
self.assert_faces(faces)
def test_get_faces_multiple(self):
oneface_img = load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))
noface_img = load_image(os.path.join(ASSETS_PATH, IMG_NOFACE))
manyface_img = load_image(os.path.join(ASSETS_PATH, IMG_MANYFACES))
imgs = [oneface_img, noface_img, manyface_img]
detection_result = self.sdk.get_faces(
imgs, qualities=True, landmarks=True, embeddings=True
)
faces = detection_result.faces
self.assertEqual(len(faces), 9, msg=ERR_UNEXPECTED_NUM_FACES)
self.assert_faces(faces)
image_inferences = detection_result.image_inferences
self.assertEqual(len(image_inferences), 3, msg=ERR_UNEXPECTED_NUM_INFERENCES)
expected_num_faces = [1, 0, 8]
for i, faces in enumerate(expected_num_faces):
self.assertEqual(
len(image_inferences[i].faces),
faces,
msg=f"unexpected number of faces found in image inference {i}",
)
def test_get_attributes(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_IDENTITY1_FACE1))]
detection_result = self.sdk.get_faces(imgs, qualities=True, landmarks=True)
faces = detection_result.faces
self.assertIsNotNone(faces, msg=ERR_MISSING_FACES)
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
face = faces[0]
self.assertIsNone(face.ages, msg=ERR_UNEXPECTED_AGES)
self.assertIsNone(face.genders, msg=ERR_UNEXPECTED_GENDERS)
self.assertIsNone(face.age, msg=ERR_UNEXPECTED_AGE)
self.assertIsNone(face.gender, msg=ERR_UNEXPECTED_GENDER)
self.sdk.get_attributes(faces)
self.assertIsNotNone(face.ages, msg="missing ages")
self.assertIsNotNone(face.genders, msg="missing genders")
self.assertIsNotNone(face.age, msg="missing age")
self.assertTrue(face.age == "20-30", msg="incorrect age")
self.assertIsNotNone(face.gender, msg="missing gender")
self.assertTrue(face.gender == "male", msg="incorrect gender")
self.assertTrue(face.ages[2] > face.ages[0], msg=ERR_INVALID_AGES)
self.assertTrue(face.ages[2] > face.ages[1], msg=ERR_INVALID_AGES)
self.assertTrue(face.ages[2] > face.ages[3], msg=ERR_INVALID_AGES)
self.assertTrue(face.ages[2] > face.ages[4], msg=ERR_INVALID_AGES)
self.assertTrue(face.ages[2] > face.ages[5], msg=ERR_INVALID_AGES)
self.assertTrue(face.ages[2] > face.ages[6], msg=ERR_INVALID_AGES)
self.assertTrue(face.genders[0] > face.genders[1], msg="invalid genders")
self.assertTrue(is_json_serializable(face.asdict()), msg=ERR_JSON_FACE)
def test_get_qualities(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_IDENTITY1_FACE1))]
faces = self.sdk.get_faces(imgs).faces
self.sdk.get_qualities(faces)
self.assertAlmostEqual(faces[0].quality, 0.925, delta=0.001)
self.assertAlmostEqual(faces[0].acceptability, 0.999, delta=0.001)
self.assertTrue(is_json_serializable(faces[0].asdict()), msg=ERR_JSON_FACE)
def test_get_faces_qualties(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_IDENTITY1_FACE1))]
faces = self.sdk.get_faces(imgs, qualities=True).faces
self.assertAlmostEqual(faces[0].quality, 0.925, delta=0.001)
self.assertTrue(is_json_serializable(faces[0].asdict()), msg=ERR_JSON_FACE)
def test_get_bounding_boxes(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
detection_result = self.sdk.get_bounding_boxes(imgs)
faces = detection_result.faces
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
f = faces[0]
self.assertIsNotNone(f.bounding_box, msg=ERR_MISSING_BBOX)
self.assertIsNotNone(f.bounding_box.score, msg=ERR_MISSING_SCORE)
self.assertIsNone(f.landmarks, msg=ERR_UNEXPECTED_LANDMARKS)
self.assertIsNone(f.quality, msg=ERR_UNEXPECTED_QUALITY)
self.assertIsNone(f.acceptability, msg="unexpected acceptability")
self.assertIsNone(f.embedding, msg="unexpected embedding")
self.assertTrue(is_json_serializable(f.asdict()), msg=ERR_JSON_FACE)
def test_get_landmarks(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
detection_result = self.sdk.get_faces(imgs)
faces = detection_result.faces
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
self.assertIsNone(faces[0].landmarks, msg=ERR_UNEXPECTED_LANDMARKS)
self.sdk.get_landmarks(faces)
self.assertIsNotNone(faces[0].landmarks, msg=ERR_MISSING_LANDMARKS)
self.assertTrue(is_json_serializable(faces[0].asdict()), msg=ERR_JSON_FACE)
def test_get_landmarks_from_bounding_box(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
detection_result = self.sdk.get_bounding_boxes(imgs)
faces = detection_result.faces
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
self.assertIsNotNone(faces[0].bounding_box, msg=ERR_MISSING_BBOX)
self.assertIsNone(faces[0].landmarks, msg=ERR_UNEXPECTED_LANDMARKS)
bbox = faces[0].bounding_box
bounding_box = BoundingBox(
bbox.origin.x,
bbox.origin.y,
bbox.origin.x + bbox.width,
bbox.origin.y + bbox.height,
)
result = self.sdk.get_landmarks_from_bounding_boxes(imgs[0], [bounding_box])
self.assertIsNotNone(result.faces[0].landmarks, msg=ERR_MISSING_LANDMARKS)
self.assertTrue(
is_json_serializable(result.faces[0].asdict()), msg=ERR_JSON_FACE
)
def test_get_embeddings(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
detection_result = self.sdk.get_faces(imgs, qualities=True, landmarks=True)
faces = detection_result.faces
self.sdk.get_embeddings(faces)
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
f = faces[0]
self.assertIsNotNone(f.bounding_box, msg=ERR_MISSING_BBOX)
self.assertIsNotNone(f.landmarks, msg=ERR_MISSING_LANDMARKS)
self.assertIsNotNone(f.embedding, msg=ERR_MISSING_EMBEDDING)
self.assertEqual(
f.embedding.scoring_mode,
ScoringMode.EnhancedEmbedding,
msg=ERR_INVALID_SCORING_MODE,
)
self.assertTrue(is_json_serializable(f.asdict()), msg=ERR_JSON_FACE)
def test_get_embedding_from_landmarks(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
detection_result = self.sdk.get_faces(imgs, embeddings=True)
faces = detection_result.faces
f = faces[0]
landmarks = f.landmarks
embeddings = self.sdk.get_embeddings_from_landmarks(
imgs[0], [landmarks, landmarks]
)
self.assertEqual(len(embeddings), 2)
embedding = embeddings[0]
self.assertTrue(embedding.scoring_mode == ScoringMode.EnhancedEmbedding)
similarity = SDK.get_similarity(f.embedding, embedding)
self.assertAlmostEqual(similarity, 1.51, delta=0.01)
def test_check_for_mask(self):
imgs = [load_image(os.path.join(ASSETS_PATH, "woman-wearing-mask.jpg"))]
detection_result = self.sdk.get_bounding_boxes(imgs)
faces = detection_result.faces
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
self.sdk.get_masks(faces)
f = faces[0]
self.assertIsNotNone(f.mask, msg=ERR_MISSING_MASK_PROB)
self.assertTrue(f.mask >= MASK_SCORE, msg=ERR_INVALID_MASK_PROB)
self.assertTrue(is_json_serializable(f.asdict()), msg=ERR_JSON_FACE)
def test_check_for_no_mask(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
detection_result = self.sdk.get_bounding_boxes(imgs)
faces = detection_result.faces
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
self.sdk.get_masks(faces)
f = faces[0]
self.assertIsNotNone(f.mask, msg=ERR_MISSING_MASK_PROB)
self.assertTrue(f.mask < MAX_NO_MASK_SCORE, msg=ERR_INVALID_MASK_PROB)
self.assertTrue(is_json_serializable(f.asdict()), msg=ERR_JSON_FACE)
def test_check_for_no_mask_in_many_faces(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_MANYFACES))]
detection_result = self.sdk.get_bounding_boxes(imgs)
faces = detection_result.faces
self.assertTrue(len(faces) > 1, msg=ERR_UNEXPECTED_NUM_FACES)
self.sdk.get_masks(faces)
for f in faces:
self.assertIsNotNone(f.mask, msg=ERR_MISSING_MASK_PROB)
self.assertTrue(f.mask < MAX_NO_MASK_SCORE, msg=ERR_INVALID_MASK_PROB)
def test_get_most_prominent_face_index_oneface(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
infer_result = self.sdk.get_bounding_boxes(imgs)
self.assertTrue(
len(infer_result.image_inferences) == 1, msg=ERR_UNEXPECTED_NUM_INFERENCES
)
self.assertNotEqual(len(infer_result.faces), 0, msg=ERR_UNEXPECTED_NUM_FACES)
infer_image = infer_result.image_inferences[0]
index = infer_image.most_prominent_face_index()
self.assertTrue(index == 0, msg=ERR_INVALID_MPF)
def test_get_most_prominent_face_index_manyfaces(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_MANYFACES))]
infer_result = self.sdk.get_bounding_boxes(imgs)
self.assertTrue(
len(infer_result.image_inferences) == 1, msg=ERR_UNEXPECTED_NUM_INFERENCES
)
self.assertTrue(len(infer_result.faces) > 0, msg=ERR_UNEXPECTED_NUM_FACES)
infer_image = infer_result.image_inferences[0]
index = infer_image.most_prominent_face_index()
self.assertTrue(index == 3, msg=ERR_INVALID_MPF)
def test_get_most_prominent_face_index_noface(self):
infer_image = ImageInferenceData(128, 128)
index = infer_image.most_prominent_face_index()
self.assertTrue(index == -1, msg=ERR_INVALID_MPF)
def test_get_most_prominent_face_index_invalid_image_dims(self):
infer_image = ImageInferenceData(0, 0)
index = infer_image.most_prominent_face_index()
self.assertTrue(index == -1, msg=ERR_INVALID_MPF)
def test_scoring_same_image(self):
img = load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))
faces = self.sdk.get_faces([img, img], embeddings=True).faces
similarity = SDK.get_similarity(faces[0].embedding, faces[1].embedding)
self.assertAlmostEqual(similarity, 1.51, delta=0.01)
confidence = self.sdk.get_confidence(faces[0].embedding, faces[1].embedding)
self.assertAlmostEqual(confidence, 1.0, delta=0.01)
match_score = SDK.get_match_score(faces[0].embedding, faces[1].embedding)
self.assertAlmostEqual(match_score, 951, delta=2)
def test_scoring_same_identity(self):
img1 = load_image(os.path.join(ASSETS_PATH, IMG_IDENTITY1_FACE1))
img2 = load_image(os.path.join(ASSETS_PATH, IMG_IDENTITY1_FACE2))
faces = self.sdk.get_faces([img1, img2], embeddings=True).faces
similarity = SDK.get_similarity(faces[0].embedding, faces[1].embedding)
self.assertAlmostEqual(similarity, 0.788, delta=0.001)
confidence = self.sdk.get_confidence(faces[0].embedding, faces[1].embedding)
self.assertAlmostEqual(confidence, 1.0, delta=0.01)
match_score = SDK.get_match_score(faces[0].embedding, faces[1].embedding)
self.assertAlmostEqual(match_score, 788, delta=2)
def test_scoring_diff_identity(self):
img1 = load_image(os.path.join(ASSETS_PATH, IMG_IDENTITY1_FACE1))
img2 = load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))
faces = self.sdk.get_faces([img1, img2], embeddings=True).faces
similarity = SDK.get_similarity(faces[0].embedding, faces[1].embedding)
self.assertAlmostEqual(similarity, 0.05, delta=0.01)
confidence = self.sdk.get_confidence(faces[0].embedding, faces[1].embedding)
self.assertAlmostEqual(confidence, 0, delta=0.01)
match_score = SDK.get_match_score(faces[0].embedding, faces[1].embedding)
self.assertAlmostEqual(match_score, 403, delta=2)
def test_get_confidence_invalid_faces(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_MANYFACES))]
faces = self.sdk.get_faces(imgs).faces
with self.assertRaises(InvalidInputException):
self.sdk.get_confidence(faces[0].embedding, faces[1].embedding)
def test_get_similarity_no_embedding(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_MANYFACES))]
faces = self.sdk.get_faces(imgs).faces
with self.assertRaises(InvalidInputException):
SDK.get_similarity(faces[0].embedding, faces[1].embedding)
def test_multi_inference_images(self):
imgs = [
load_image(os.path.join(ASSETS_PATH, IMG_MANYFACES)),
load_image(os.path.join(ASSETS_PATH, IMG_MANYFACES)),
load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE)),
]
infer_result = self.sdk.get_bounding_boxes(imgs)
self.assertTrue(
len(infer_result.image_inferences) == 3, msg=ERR_UNEXPECTED_NUM_INFERENCES
)
self.assertTrue(
len(infer_result.image_inferences[0].faces)
+ len(infer_result.image_inferences[1].faces)
+ len(infer_result.image_inferences[2].faces)
== len(infer_result.faces),
msg="inference image data mismatches faces len",
)
def test_inference_image_data(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
infer_result = self.sdk.get_bounding_boxes(imgs)
faces = infer_result.faces
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
self.sdk.get_qualities(faces)
self.assertAlmostEqual(faces[0].quality, 0.895, delta=0.001)
self.assertTrue(
infer_result.image_inferences[0].faces[0].quality == faces[0].quality,
msg="image inference data and face mismatch",
)
def test_check_embedding(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
ground_truth = np.load(
os.path.join(ASSETS_PATH, "oneface_gen5_fast_enhanced_embedding.npy")
)
detection_result = self.sdk.get_faces(imgs, qualities=True, landmarks=True)
faces = detection_result.faces
self.sdk.get_embeddings(faces)
self.assertEqual(len(faces), 1, msg=ERR_UNEXPECTED_NUM_FACES)
f = faces[0]
self.assertEqual(
len(f.embedding.data), len(ground_truth), msg="Mismatched embedding size"
)
self.assertTrue(
np.allclose(f.embedding.data, ground_truth, rtol=0, atol=35e-4),
msg="Invalid embedding value",
)
self.assertTrue(is_json_serializable(f.asdict()), msg=ERR_JSON_FACE)
def test_get_embedding_from_prepared_image(self):
imgs = [load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE))]
detection_result = self.sdk.get_faces(imgs, embeddings=True)
faces = detection_result.faces
f = faces[0]
reco_img = load_image(os.path.join(ASSETS_PATH, IMG_ONEFACE_RECO_INPUT_IMG))
embedding = self.sdk.get_embedding_from_prepared_image(reco_img)
self.assertTrue(len(embedding.data) == EXPECTED_ENHANCED_EMBED_LEN)
self.assertTrue(embedding.scoring_mode == scoring_mode)
self.assertTrue(
np.allclose(f.embedding.data, embedding.data, rtol=0, atol=0.001),
msg="Invalid embedding value",
)
def test_get_embedding_from_prepared_image_none(self):
with self.assertRaises(InvalidInputException):
self.sdk.get_embedding_from_prepared_image(None)
def assert_faces(self, faces):
for f in faces:
self.assertIsNotNone(f.bounding_box, msg=ERR_MISSING_BBOX)
self.assertIsNotNone(f.landmarks, msg=ERR_MISSING_LANDMARKS)
self.assertIsNotNone(f.quality, msg="missing quality")
self.assertIsNotNone(f.acceptability, msg="missing acceptability")
self.assertIsNotNone(
f.recognition_input_image, msg="missing recognition input image"
)
self.assertIsNotNone(
f.landmarks_input_image, msg="missing landmarks input image"
)
self.assertIsNotNone(
f.landmarks_input_bounding_box,
msg="missing landmarks input bounding box",
)
self.assertIsNotNone(f.alignment_image, msg="missing alignment image")
self.assertIsNotNone(
f.alignment_bounding_box, msg="missing alignment bounding box"
)
self.assertIsNotNone(f.embedding, msg=ERR_MISSING_EMBEDDING)
self.assertEqual(
f.embedding.scoring_mode,
ScoringMode.EnhancedEmbedding,
msg=ERR_INVALID_SCORING_MODE,
)
self.assertTrue(
len(f.embedding.data) in Embedding.ENHANCED_SIZES,
msg=ERR_INVALID_EMBEDDING_SIZE,
)
self.assertIsNone(f.ages, msg=ERR_UNEXPECTED_AGES)
self.assertIsNone(f.genders, msg=ERR_UNEXPECTED_GENDERS)
self.assertIsNone(f.age, msg=ERR_UNEXPECTED_AGE)
self.assertIsNone(f.gender, msg=ERR_UNEXPECTED_GENDER)
self.assertTrue(is_json_serializable(f.asdict()), msg=ERR_JSON_FACE)