From 972e5e1a8acbdf1763212c4a0278b2ec544893db Mon Sep 17 00:00:00 2001 From: Shawn-Shan Date: Wed, 29 Jul 2020 01:50:42 -0500 Subject: [PATCH] prepare for 0.3 release --- fawkes/__main__.py | 6 ++++ fawkes/align_face.py | 32 +++++++++++++++-- fawkes/detect_faces.py | 24 ++++++++++++- fawkes/differentiator.py | 29 ++++----------- fawkes/master.py | 66 ++++++++++++++++++++++++++++++++++ fawkes/protection.py | 77 ++++++++++++++++++++-------------------- fawkes/utils.py | 65 ++++++++++++--------------------- 7 files changed, 193 insertions(+), 106 deletions(-) create mode 100644 fawkes/master.py diff --git a/fawkes/__main__.py b/fawkes/__main__.py index b8315cf..a8771ad 100644 --- a/fawkes/__main__.py +++ b/fawkes/__main__.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# @Date : 2020-07-01 +# @Author : Shawn Shan (shansixiong@cs.uchicago.edu) +# @Link : https://www.shawnshan.com/ + + from .protection import main if __name__ == '__main__': diff --git a/fawkes/align_face.py b/fawkes/align_face.py index 59e52f2..bc30878 100644 --- a/fawkes/align_face.py +++ b/fawkes/align_face.py @@ -1,3 +1,31 @@ +"""Performs face alignment and stores face thumbnails in the output directory.""" +# MIT License +# +# Copyright (c) 2016 David Sandberg +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" Tensorflow implementation of the face detection / alignment algorithm found at +https://github.com/kpzhang93/MTCNN_face_detection_alignment +""" + + import numpy as np from fawkes import create_mtcnn, run_detect_face @@ -19,8 +47,8 @@ def aligner(sess): def align(orig_img, aligner, margin=0.8, detect_multiple_faces=True): pnet, rnet, onet = aligner - minsize = 20 # minimum size of face - threshold = [0.6, 0.7, 0.7] # three steps's threshold + minsize = 25 # minimum size of face + threshold = [0.85, 0.85, 0.85] # three steps's threshold factor = 0.709 # scale factor if orig_img.ndim < 2: diff --git a/fawkes/detect_faces.py b/fawkes/detect_faces.py index 9d6f317..ac40163 100644 --- a/fawkes/detect_faces.py +++ b/fawkes/detect_faces.py @@ -1,3 +1,26 @@ +"""Performs face alignment and stores face thumbnails in the output directory.""" +# MIT License +# +# Copyright (c) 2016 David Sandberg +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + """ Tensorflow implementation of the face detection / alignment algorithm found at https://github.com/kpzhang93/MTCNN_face_detection_alignment """ @@ -9,7 +32,6 @@ import pickle import numpy as np import tensorflow as tf from six import string_types, iteritems -from skimage.transform import resize def layer(op): diff --git a/fawkes/differentiator.py b/fawkes/differentiator.py index f84352a..3a6651a 100644 --- a/fawkes/differentiator.py +++ b/fawkes/differentiator.py @@ -161,7 +161,13 @@ class FawkesMaskGeneration: def calculate_direction(bottleneck_model, cur_timg_input, cur_simg_input): target_features = bottleneck_model(cur_timg_input) - return target_features + # return target_features + target_center = tf.reduce_mean(target_features, axis=0) + original = bottleneck_model(cur_simg_input) + original_center = tf.reduce_mean(original, axis=0) + direction = target_center - original_center + final_target = original + 2.0 * direction + return final_target self.bottlesim = 0.0 self.bottlesim_sum = 0.0 @@ -280,14 +286,9 @@ class FawkesMaskGeneration: def attack_batch(self, source_imgs, target_imgs, weights): - """ - Run the attack on a batch of images and labels. - """ - LR = self.learning_rate nb_imgs = source_imgs.shape[0] mask = [True] * nb_imgs + [False] * (self.batch_size - nb_imgs) - # mask = [True] * self.batch_size mask = np.array(mask, dtype=np.bool) source_imgs = np.array(source_imgs) @@ -324,16 +325,6 @@ class FawkesMaskGeneration: self.assign_mask: mask, self.assign_weights: weights_batch, self.assign_modifier: modifier_batch}) - else: - # if directly mimicking a vector, use assign_bottleneck_t_raw - # in setup - self.sess.run(self.setup, - {self.assign_bottleneck_t_raw: timg_tanh_batch, - self.assign_simg_tanh: simg_tanh_batch, - self.assign_const: CONST, - self.assign_mask: mask, - self.assign_weights: weights_batch, - self.assign_modifier: modifier_batch}) best_bottlesim = [0] * nb_imgs if self.maximize else [np.inf] * nb_imgs best_adv = np.zeros_like(source_imgs) @@ -394,12 +385,6 @@ class FawkesMaskGeneration: best_bottlesim[e] = bottlesim best_adv[e] = aimg_input - # if iteration > 20 and (dist_raw >= self.l_threshold or iteration == self.MAX_ITERATIONS - 1): - # finished_idx.add(e) - # print("{} finished at dist {}".format(e, dist_raw)) - # best_bottlesim[e] = bottlesim - # best_adv[e] = aimg_input - # all_clear = False if all_clear: diff --git a/fawkes/master.py b/fawkes/master.py new file mode 100644 index 0000000..24a50f1 --- /dev/null +++ b/fawkes/master.py @@ -0,0 +1,66 @@ +import socket +import subprocess +import sys +import time + +print(socket.gethostname()) + + +def assign_gpu(args, gpu_idx): + for i, arg in enumerate(args): + if arg == "GPUID": + args[i] = str(gpu_idx) + return args + + +def produce_present(): + process_ls = [] + gpu_ls = list(sys.argv[1]) + max_num = int(sys.argv[2]) + + available_gpus = [] + i = 0 + while len(available_gpus) < max_num: + if i > len(gpu_ls) - 1: + i = 0 + available_gpus.append(gpu_ls[i]) + i += 1 + + process_dict = {} + all_queries_to_run = [] + + for m in ['mid', 'low', 'min']: + for directory in ['KimKardashian', 'Liuyifei', 'Obama', 'TaylorSwift', 'TomHolland']: + args = ['python3', 'protection.py', '--gpu', 'GPUID', '-d', + '/home/shansixioing/fawkes/data/test/{}/'.format(directory), + '--batch-size', '30', '-m', m, + '--debug'] + args = [str(x) for x in args] + all_queries_to_run.append(args) + + for args in all_queries_to_run: + cur_gpu = available_gpus.pop(0) + args = assign_gpu(args, cur_gpu) + print(" ".join(args)) + p = subprocess.Popen(args) + process_ls.append(p) + process_dict[p] = cur_gpu + + gpu_ls.append(cur_gpu) + time.sleep(5) + while not available_gpus: + for p in process_ls: + poll = p.poll() + if poll is not None: + process_ls.remove(p) + available_gpus.append(process_dict[p]) + + time.sleep(20) + + +def main(): + produce_present() + + +if __name__ == '__main__': + main() diff --git a/fawkes/protection.py b/fawkes/protection.py index 0b8c04a..3e6c209 100644 --- a/fawkes/protection.py +++ b/fawkes/protection.py @@ -1,6 +1,8 @@ -# from __future__ import absolute_import -# from __future__ import division -# from __future__ import print_function +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Date : 2020-05-17 +# @Author : Shawn Shan (shansixiong@cs.uchicago.edu) +# @Link : https://www.shawnshan.com/ import argparse import glob @@ -54,14 +56,18 @@ class Fawkes(object): self.protector_param = None def mode2param(self, mode): - if mode == 'low': - th = 0.0025 - max_step = 30 - lr = 30 + if mode == 'min': + th = 0.002 + max_step = 20 + lr = 40 + elif mode == 'low': + th = 0.003 + max_step = 50 + lr = 35 elif mode == 'mid': th = 0.005 - max_step = 100 - lr = 15 + max_step = 200 + lr = 20 elif mode == 'high': th = 0.008 max_step = 500 @@ -77,7 +83,7 @@ class Fawkes(object): raise Exception("mode must be one of 'low', 'mid', 'high', 'ultra', 'custom'") return th, max_step, lr - def run_protection(self, image_paths, mode='low', th=0.04, sd=1e9, lr=10, max_step=500, batch_size=1, format='png', + def run_protection(self, image_paths, mode='min', th=0.04, sd=1e9, lr=10, max_step=500, batch_size=1, format='png', separate_target=True, debug=False): if mode == 'custom': pass @@ -137,10 +143,6 @@ class Fawkes(object): faces.cloaked_cropped_faces = protected_images - # cloak_perturbation = reverse_process_cloaked(protected_images) - reverse_process_cloaked( - # original_images) - # final_images = faces.merge_faces(cloak_perturbation) - final_images = faces.merge_faces(reverse_process_cloaked(protected_images), reverse_process_cloaked(original_images)) @@ -164,28 +166,34 @@ def main(*argv): parser = argparse.ArgumentParser() parser.add_argument('--directory', '-d', type=str, - help='directory that contain images for cloaking', default='imgs/') + help='the directory that contains images to run protection', default='imgs/') parser.add_argument('--gpu', '-g', type=str, - help='GPU id', default='0') + help='the GPU id when using GPU for optimization', default='0') parser.add_argument('--mode', '-m', type=str, - help='cloak generation mode', default='low') + help='cloak generation mode, select from min, low, mid, high. The higher the mode is, the more perturbation added and stronger protection', + default='min') + parser.add_argument('--feature-extractor', type=str, - help="name of the feature extractor used for optimization", + help="name of the feature extractor used for optimization, currently only support high_extract", default="high_extract") - parser.add_argument('--th', type=float, default=0.01) - parser.add_argument('--max-step', type=int, default=1000) - parser.add_argument('--sd', type=int, default=1e9) - parser.add_argument('--lr', type=float, default=2) - - parser.add_argument('--batch-size', type=int, default=1) - parser.add_argument('--separate_target', action='store_true') - parser.add_argument('--debug', action='store_true') + parser.add_argument('--th', help='only relevant with mode=custom, DSSIM threshold for perturbation', type=float, + default=0.01) + parser.add_argument('--max-step', help='only relevant with mode=custom, number of steps for optimization', type=int, + default=1000) + parser.add_argument('--sd', type=int, help='only relevant with mode=custom, penalty number, read more in the paper', + default=1e6) + parser.add_argument('--lr', type=float, help='only relevant with mode=custom, learning rate', default=2) + parser.add_argument('--batch-size', help="number of images to run optimization together", type=int, default=1) + parser.add_argument('--separate_target', help="whether select separate targets for each faces in the directory", + action='store_true') + parser.add_argument('--debug', help="turn on debug and copy/paste the stdout when reporting an issue on github", + action='store_true') parser.add_argument('--format', type=str, - help="final image format", + help="format of the output image", default="png") args = parser.parse_args(argv[1:]) @@ -198,17 +206,10 @@ def main(*argv): image_paths = [path for path in image_paths if "_cloaked" not in path.split("/")[-1]] protector = Fawkes(args.feature_extractor, args.gpu, args.batch_size) - if args.mode != 'all': - protector.run_protection(image_paths, mode=args.mode, th=args.th, sd=args.sd, lr=args.lr, - max_step=args.max_step, - batch_size=args.batch_size, format=args.format, - separate_target=args.separate_target, debug=args.debug) - else: - for m in ['low', 'mid', 'high']: - protector.run_protection(image_paths, mode=m, th=args.th, sd=args.sd, lr=args.lr, - max_step=args.max_step, - batch_size=args.batch_size, format=args.format, - separate_target=args.separate_target, debug=args.debug) + protector.run_protection(image_paths, mode=args.mode, th=args.th, sd=args.sd, lr=args.lr, + max_step=args.max_step, + batch_size=args.batch_size, format=args.format, + separate_target=args.separate_target, debug=args.debug) if __name__ == '__main__': diff --git a/fawkes/utils.py b/fawkes/utils.py index 1a867b2..843a4d2 100644 --- a/fawkes/utils.py +++ b/fawkes/utils.py @@ -1,3 +1,10 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Date : 2020-05-17 +# @Author : Shawn Shan (shansixiong@cs.uchicago.edu) +# @Link : https://www.shawnshan.com/ + + import errno import glob import gzip @@ -23,10 +30,9 @@ import keras.backend as K import numpy as np import tensorflow as tf from PIL import Image, ExifTags -from keras.layers import Dense, Activation, Dropout +from keras.layers import Dense, Activation from keras.models import Model from keras.preprocessing import image -# from skimage.transform import resize from fawkes.align_face import align from six.moves.urllib.request import urlopen @@ -72,7 +78,12 @@ def load_image(path): except IsADirectoryError: return None - if img._getexif() is not None: + try: + info = img._getexif() + except OSError: + return None + + if info is not None: for orientation in ExifTags.TAGS.keys(): if ExifTags.TAGS[orientation] == 'Orientation': break @@ -109,7 +120,7 @@ def filter_image_paths(image_paths): class Faces(object): - def __init__(self, image_paths, loaded_images, aligner, verbose=1, eval_local=False): + def __init__(self, image_paths, loaded_images, aligner, verbose=1, eval_local=False, preprocessing=True): self.image_paths = image_paths self.verbose = verbose self.aligner = aligner @@ -165,7 +176,8 @@ class Faces(object): self.cropped_faces = np.array(self.cropped_faces) - self.cropped_faces = preprocess(self.cropped_faces, 'imagenet') + if preprocessing: + self.cropped_faces = preprocess(self.cropped_faces, 'imagenet') self.cloaked_cropped_faces = None self.cloaked_faces = np.copy(self.org_faces) @@ -178,14 +190,12 @@ class Faces(object): self.cloaked_faces = np.copy(self.org_faces) for i in range(len(self.cropped_faces)): - # cur_cloak = cloaks[i] cur_protected = protected_images[i] cur_original = original_images[i] org_shape = self.cropped_faces_shape[i] old_square_shape = max([org_shape[0], org_shape[1]]) - # reshape_cloak = resize(cur_cloak, (old_square_shape, old_square_shape)) cur_protected = resize(cur_protected, (old_square_shape, old_square_shape)) cur_original = resize(cur_original, (old_square_shape, old_square_shape)) @@ -197,6 +207,8 @@ class Faces(object): bb = self.cropped_index[i] self.cloaked_faces[callback_id][bb[1]:bb[3], bb[0]:bb[2], :] += reshape_cloak + for i in range(0, len(self.cloaked_faces)): + self.cloaked_faces[i] = np.clip(self.cloaked_faces[i], 0.0, 255.0) return self.cloaked_faces @@ -206,12 +218,11 @@ def dump_dictionary_as_json(dict, outfile): f.write(j.encode()) -def load_victim_model(number_classes, teacher_model=None, end2end=False, dropout=0): +def load_victim_model(number_classes, teacher_model=None, end2end=False): for l in teacher_model.layers: l.trainable = end2end x = teacher_model.layers[-1].output - if dropout > 0: - x = Dropout(dropout)(x) + x = Dense(number_classes)(x) x = Activation('softmax', name="act")(x) model = Model(teacher_model.input, x) @@ -412,29 +423,12 @@ def get_dataset_path(dataset): 'num_images'] -def normalize(x): - return x / np.linalg.norm(x, axis=1, keepdims=True) - - def dump_image(x, filename, format="png", scale=False): - # img = image.array_to_img(x, scale=scale) - img = image.array_to_img(x) + img = image.array_to_img(x, scale=scale) img.save(filename, format) return -def load_dir(path): - assert os.path.exists(path) - x_ls = [] - for file in os.listdir(path): - cur_path = os.path.join(path, file) - im = image.load_img(cur_path, target_size=(224, 224)) - im = image.img_to_array(im) - x_ls.append(im) - raw_x = np.array(x_ls) - return preprocess(raw_x, 'imagenet') - - def load_embeddings(feature_extractors_names): model_dir = os.path.join(os.path.expanduser('~'), '.fawkes') dictionaries = [] @@ -457,7 +451,6 @@ def extractor_ls_predict(feature_extractors_ls, X): cur_features = extractor.predict(X) feature_ls.append(cur_features) concated_feature_ls = np.concatenate(feature_ls, axis=1) - concated_feature_ls = normalize(concated_feature_ls) return concated_feature_ls @@ -477,20 +470,6 @@ def pairwise_l2_distance(A, B): return ED -def calculate_dist_score(a, b, feature_extractors_ls, metric='l2'): - features1 = extractor_ls_predict(feature_extractors_ls, a) - features2 = extractor_ls_predict(feature_extractors_ls, b) - - pair_cos = pairwise_l2_distance(features1, features2) - max_sum = np.min(pair_cos, axis=0) - max_sum_arg = np.argsort(max_sum)[::-1] - max_sum_arg = max_sum_arg[:len(a)] - max_sum = [max_sum[i] for i in max_sum_arg] - paired_target_X = [b[j] for j in max_sum_arg] - paired_target_X = np.array(paired_target_X) - return np.min(max_sum), paired_target_X - - def select_target_label(imgs, feature_extractors_ls, feature_extractors_names, metric='l2'): model_dir = os.path.join(os.path.expanduser('~'), '.fawkes')