Multymodel

.gitignore

@@ -12,6 +12,7 @@ build/
 develop-eggs/
 dist/
 downloads/
+runs/
 eggs/
 .eggs/
 lib64/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "RAdam"]
+	path = RAdam
+	url = https://github.com/LiyuanLucasLiu/RAdam.git

argument_parsing.py

@@ -36,7 +36,7 @@ def parse_command_line_args_train():
         help='percentage of validation data',
         required=False,
         type=float,
-        default=0.05,
+        default=1,
     )
     parser.add_argument(
         '--batch_size',

attack_construction/attack_class.py

@@ -0,0 +1,211 @@
+#from turtle import forward
+import torch
+import torch.nn as nn
+import numpy as np
+import torch.nn.functional as F
+import attack_construction.attack_methods as attack
+from torch.utils.tensorboard import SummaryWriter
+from attack_construction.utils import save_patch_tensor
+from progress.bar import IncrementalBar
+import json
+from pycocotools.coco import COCO
+from pycocotools.cocoeval import COCOeval
+
+
+def train(my_complex_model, train_dataloader, augmentations, optimizer, writer, loss, epoch):
+    bar = IncrementalBar(f'Epoch {epoch} progress', max = len(train_dataloader))
+
+    for step_num, (images, labels, _, _) in enumerate(train_dataloader):
+        for model_index in range(len(my_complex_model.module.models)):
+            prediction = my_complex_model(images, labels, model_index, augmentations)
+            costs = loss(prediction, my_complex_model.module.patch)
+            cost = sum(costs)
+            cost.backward()
+
+        optimizer.step()
+        optimizer.zero_grad()
+
+        with torch.no_grad():
+            my_complex_model.module.patch.data.clamp_(0,1)
+
+        bar.next()
+
+    bar.finish()
+
+
+#в разработке
+def validate(my_complex_model, val_dataloader, augmentations, annotation_file, local_rank):
+    if (local_rank != 0):
+        return
+
+    mAPs = []
+
+    for model_index in range(len(my_complex_model.module.models)):
+        annotation_after = []
+
+        bar = IncrementalBar(f'Validate {model_index} model in progress progress', max = len(val_dataloader))
+
+        for val_idx, (images, labels, img_ids, scale_factor) in enumerate(val_dataloader):
+            with torch.no_grad():
+                prediction = my_complex_model(images, labels, model_index, augmentations)
+
+            for i in range(len(prediction)):
+                for j in range(len(prediction[i]["labels"])):
+                    annotation_after.append({
+                        'image_id': img_ids[i].item(),
+                        'category_id': prediction[i]["labels"][j].item(),
+                        'bbox': [
+                            prediction[i]["boxes"][j][0].item() / scale_factor[i][0].item(),
+                            prediction[i]["boxes"][j][1].item() / scale_factor[i][1].item(),
+                            (prediction[i]["boxes"][j][2].item() - prediction[i]["boxes"][j][0].item()) / scale_factor[i][0].item(),
+                            (prediction[i]["boxes"][j][3].item() - prediction[i]["boxes"][j][1].item()) / scale_factor[i][1].item()
+                        ],
+                        "score": prediction[i]['scores'][j].item()
+                    })
+
+            bar.next()
+
+        bar.finish()
+
+        with open("tmp.json", 'w') as f_after:
+            json.dump(annotation_after, f_after)
+
+        cocoGt = COCO(annotation_file)
+        cocoDt = cocoGt.loadRes("./tmp.json")
+
+        cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
+        cocoEval.params.imgIds = cocoGt.getImgIds()
+        cocoEval.evaluate()
+        cocoEval.accumulate()
+        cocoEval.summarize()
+
+        print("mAP =", np.mean(cocoEval.stats))
+        mAPs.append(np.mean(cocoEval.stats))
+
+
+    return mAPs 
+
+
+
+
+class Attack_class(nn.Module):
+    def __init__(self, models, patch, local_rank):
+        super().__init__()
+        self.models = models
+        self.patch = nn.Parameter(data=patch)
+        self.local_rank = local_rank
+
+
+    def forward(self, images, labels, model_index, augmentations):
+        if (model_index >= len(self.models)):
+            return None
+
+        augmented_patch = self.patch if augmentations is None else augmentations(self.patch)
+
+        attacked_images = []
+
+        for i, image in enumerate(images):
+            attacked_image = image.to(f'cuda:{self.local_rank}')
+
+            if labels[i][0][2] * labels[i][0][3] != 0:
+                for label in labels[i]:
+                    attacked_image = attack.insert_patch(attacked_image, augmented_patch, label, 0.4, self.local_rank, True)
+
+            attacked_images.append(attacked_image)
+
+        if (len(attacked_images) == 0):
+            return None
+
+        return self.models[model_index](attacked_images)
+
+
+
+
+
+
+
+
+
+    #ниже устаревшее
+'''
+    def train(self, epochs, train_loader, batch_size, augmentations, loss_function, optimizer, experiment_dir, step_save_frequency, val_loader, small_val_loader, val_labels):
+        writer = SummaryWriter(log_dir=experiment_dir.as_posix())
+        for epoch in range(epochs):
+            image_counter = 0
+            prev_steps = epoch * len(train_loader)
+
+            for step_num, (images, labels, _, _) in enumerate(train_loader):
+                image_counter += batch_size
+
+                losses = []
+                for model in self.models:
+                    loss, patch = attack.training_step(
+                        model=model,
+                        patch=self.patch,
+                        augmentations=augmentations,
+                        images=images,
+                        labels=labels,
+                        loss=loss_function,
+                        device=self.device,
+                        optimizer = optimizer,
+                    )
+                    losses.append(loss)
+                    self.patch = patch
+                loss = np.mean(losses)
+
+                # TODO: apply tqdm library for progress logging
+                print(f"ep:{epoch}, epoch_progress:{step_num/len(train_loader)}, batch_loss:{loss}")
+                writer.add_scalar('Loss/train', loss, step_num + prev_steps)
+
+                if step_num % step_save_frequency == 0:
+                    self.log_results(writer, experiment_dir, epoch, step_num, prev_steps, small_val_loader, val_labels, augmentations)
+
+            # at least one time in epoch you need full validation
+            self.log_results(writer, experiment_dir, epoch, step_num, prev_steps, val_loader, val_labels, augmentations)
+
+        writer.close()
+
+
+    def validate(self, validate_dir, val_loader, val_labels, augmentations):
+        objs = 0
+        tvs = 0
+        maps = 0
+        for model in self.models:
+            obj, tv, mAP = attack.validate(
+                model, 
+                self.patch, 
+                augmentations, 
+                val_loader, 
+                self.device, 
+                val_labels, 
+                validate_dir)
+            objs += obj
+            tvs += tv
+            maps += mAP
+
+        obj = objs / len(self.models)
+        tv = tvs / len(self.models)
+        mAP = maps / len(self.models)
+
+        print(f'Patch validated. VAL: objectness:{obj}, tv:{tv}, mAP:{mAP}')
+        return obj, tv, mAP
+
+
+    def log_results(self, writer, experiment_dir, epoch, step_num, prev_steps, small_val_loader, val_labels, augmentations):
+        save_patch_tensor(self.patch, experiment_dir, epoch=epoch, step=step_num, save_mode='both')
+        validate_dir = experiment_dir / ('validate_epoch_' + str(epoch) + '_step_' + str(step_num))
+        validate_dir.mkdir(parents=True, exist_ok=True)
+        obj, tv, mAP = self.validate(
+                validate_dir,
+                small_val_loader, 
+                val_labels, 
+                augmentations)
+        print(f'patch saved. VAL: objectness:{obj}, attacked:{tv}, mAP:{mAP}')
+        writer.add_scalar('Loss/val_obj', obj, step_num + prev_steps)
+        writer.add_scalar('Loss/val_tv', tv, step_num + prev_steps)
+        writer.add_scalar('mAP/val', mAP, step_num + prev_steps)
+        writer.flush()
+
+
+'''
+

attack_construction/attack_methods.py

@@ -15,26 +15,27 @@
 import attack_construction.metrics as metrics
 
 
-def adversarial_loss_function_batch(predicts, patch, device, tv_scale):
-    return [adversarial_loss_function(predict, patch, device, tv_scale) for predict in predicts]
+def adversarial_loss_function_batch(predicts, patch, tv_scale, local_rank):
+    return [adversarial_loss_function(predict, patch, tv_scale, local_rank) for predict in predicts]
 
 
-def adversarial_loss_function(predict, patch, device, tv_scale):
-    return metrics.general_objectness(predict, device) + tv_scale * metrics.total_variation(patch)
+def adversarial_loss_function(predict, patch,tv_scale, local_rank):
+    return metrics.general_objectness(predict, local_rank) + tv_scale * metrics.total_variation(patch)
 
 
-def generate_random_patch(resolution=(200, 200)):
+def generate_random_patch(resolution=(90, 90)):
     return torch.rand(3, resolution[0], resolution[1])
 
 
-def insert_patch(image, patch, box, ratio, device, random_place=False):
+def insert_patch(image, patch, box, ratio, local_rank ,random_place=False):
     patch_size = (int(box[3] * ratio), int(box[2] * ratio))
 
     # cant insert patch with this box parameters
     if patch_size[0] == 0 or patch_size[1] == 0:
         return image
 
     resized_patch = T.Resize(size=patch_size, interpolation=T.InterpolationMode.BICUBIC)(patch)
+    resized_patch = torch.clamp(resized_patch, 0, 1)
 
     patch_x_offset = box[2] * (random.uniform(0, 1 - ratio) if random_place else 0.5 - ratio / 2)
     patch_y_offset = box[3] * (random.uniform(0.3, 1 - ratio) if random_place else 0.5 - ratio / 2)
@@ -43,16 +44,13 @@ def insert_patch(image, patch, box, ratio, device, random_place=False):
 
     padding = (x_shift, y_shift, image.shape[2] - x_shift - patch_size[1], image.shape[1] - y_shift - patch_size[0])
     padded_patch = T.Pad(padding=padding)(resized_patch)
-    patch_mask = T.Pad(padding=padding)(torch.ones(size=(3, patch_size[0], patch_size[1]))).to(device)
+    patch_mask = T.Pad(padding=padding)(torch.ones(size=(3, patch_size[0], patch_size[1]))).to(f'cuda:{local_rank}')
     result = padded_patch + (torch.ones_like(image) - patch_mask) * image
     return result
 
 
-def training_step(model, patch, augmentations, images, labels, loss, device, grad_rate):
-    torch.cuda.empty_cache()
-
-    patch.requires_grad = True
-
+def training_step(model, patch, augmentations, images, labels, loss, device, optimizer):
+
     attacked_images = [] #torch.tensor(image.to(device), requires_grad = True) for image in images
 
     augmented_patch = patch if augmentations is None else augmentations(patch)
@@ -67,25 +65,64 @@ def training_step(model, patch, augmentations, images, labels, loss, device, gra
             attacked_images.append(attacked_image)
 
     costMean = 0
-
+    
     if len(attacked_images) != 0:
 
         predict = model(attacked_images)
 
         costs = loss(predict, patch, device)
-        grad = torch.autograd.grad(outputs=sum(costs), inputs=patch, retain_graph=True, create_graph=True, allow_unused=True)[0]
-
-        if grad is not None:
-            patch = torch.clamp(patch - grad_rate * grad.sign(), 0, 1)
 
+        cost = sum(costs)
+
+        cost.backward()
 
-        costMean = np.mean(np.asarray([cost.detach().cpu().numpy() for cost in costs]))
+        optimizer.step()
+        optimizer.zero_grad()
+
+        with torch.no_grad():
+            patch.data.clamp_(0,1)
 
+        costMean = np.mean(np.asarray([cost.detach().cpu().numpy() for cost in costs]))
 
-    patch = patch.detach() 
     return costMean, patch
 
 
+def training_step_multymodels(models, patch, augmentations, images, labels, loss, device, optimizer):
+    attacked_images = [] #torch.tensor(image.to(device), requires_grad = True) for image in images
+
+    augmented_patch = patch if augmentations is None else augmentations(patch)
+
+    for i, image in enumerate(images):
+        if labels[i][0][2] * labels[i][0][3] != 0:
+            attacked_image = image.to(device)
+
+            for label in labels[i]:
+                attacked_image = insert_patch(attacked_image, augmented_patch, label, 0.4, device, model.model.local_rank, True)
+
+            attacked_images.append(attacked_image)
+
+    costMean = 0
+
+    if len(attacked_images) != 0:
+        costMean = []
+
+        for model in models:
+            predict = model(attacked_images)
+            costs = loss(predict, patch, device)
+            cost = sum(costs)
+            cost.backward()
+            optimizer.step()
+
+            with torch.no_grad():
+                patch.data.clamp_(0,1)
+
+            costMean.append(np.mean(np.asarray([cost.detach().cpu().numpy() for cost in costs])))
+
+
+    optimizer.zero_grad()
+    return np.mean(costMean), patch
+
+
 def validate(
         model,
         patch,
@@ -110,7 +147,7 @@ def validate(
         if augmented_patch is not None:
             for i, _ in enumerate(images):
                 for label in labels[i]:
-                    attacked_images[i] = insert_patch(attacked_images[i], augmented_patch, label, 0.4, device, True)
+                    attacked_images[i] = insert_patch(attacked_images[i], augmented_patch, label, 0.4, device, model.model.local_rank , True)
 
         with torch.no_grad():
             predict = model(attacked_images)
@@ -156,4 +193,4 @@ def validate(
     if patch is not None:
         total_variation = attack_metric.total_variation(patch).detach().cpu()
 
-    return np.mean(np.asarray(objectness)), total_variation, np.mean(cocoEval.stats)
+    return np.mean(np.asarray(objectness)), total_variation, np.mean(cocoEval.stats)