246 lines
11 KiB
Python
246 lines
11 KiB
Python
from itertools import chain
|
|
from math import ceil
|
|
from pathlib import Path
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
import torchvision
|
|
|
|
import ignite.distributed as idist
|
|
from ignite.engine import Events, Engine
|
|
from ignite.metrics import RunningAverage
|
|
from ignite.utils import convert_tensor
|
|
from ignite.contrib.handlers.tensorboard_logger import OptimizerParamsHandler
|
|
from ignite.contrib.handlers.param_scheduler import PiecewiseLinear
|
|
|
|
from omegaconf import OmegaConf, read_write
|
|
|
|
import data
|
|
from loss.gan import GANLoss
|
|
from model.weight_init import generation_init_weights
|
|
from model.GAN.residual_generator import GANImageBuffer
|
|
from loss.I2I.edge_loss import EdgeLoss
|
|
from loss.I2I.perceptual_loss import PerceptualLoss
|
|
from util.image import make_2d_grid
|
|
from util.handler import setup_common_handlers, setup_tensorboard_handler
|
|
from util.build import build_model, build_optimizer
|
|
|
|
|
|
def build_lr_schedulers(optimizers, config):
|
|
g_milestones_values = [
|
|
(0, config.optimizers.generator.lr),
|
|
(int(config.data.train.scheduler.start_proportion * config.max_iteration), config.optimizers.generator.lr),
|
|
(config.max_iteration, config.data.train.scheduler.target_lr)
|
|
]
|
|
d_milestones_values = [
|
|
(0, config.optimizers.discriminator.lr),
|
|
(int(config.data.train.scheduler.start_proportion * config.max_iteration), config.optimizers.discriminator.lr),
|
|
(config.max_iteration, config.data.train.scheduler.target_lr)
|
|
]
|
|
return dict(
|
|
g=PiecewiseLinear(optimizers["g"], param_name="lr", milestones_values=g_milestones_values),
|
|
d=PiecewiseLinear(optimizers["d"], param_name="lr", milestones_values=d_milestones_values)
|
|
)
|
|
|
|
|
|
def get_trainer(config, logger, train_data_loader):
|
|
generator = build_model(config.model.generator, config.distributed.model)
|
|
discriminators = dict(
|
|
a=build_model(config.model.discriminator, config.distributed.model),
|
|
b=build_model(config.model.discriminator, config.distributed.model),
|
|
)
|
|
generation_init_weights(generator)
|
|
for m in discriminators.values():
|
|
generation_init_weights(m)
|
|
|
|
logger.debug(discriminators["a"])
|
|
logger.debug(generator)
|
|
|
|
optimizers = dict(
|
|
g=build_optimizer(generator.parameters(), config.optimizers.generator),
|
|
d=build_optimizer(chain(*[m.parameters() for m in discriminators.values()]), config.optimizers.discriminator),
|
|
)
|
|
logger.info(f"build optimizers:\n{optimizers}")
|
|
|
|
lr_schedulers = build_lr_schedulers(optimizers, config)
|
|
logger.info(f"build lr_schedulers:\n{lr_schedulers}")
|
|
|
|
gan_loss_cfg = OmegaConf.to_container(config.loss.gan)
|
|
gan_loss_cfg.pop("weight")
|
|
gan_loss = GANLoss(**gan_loss_cfg).to(idist.device())
|
|
|
|
edge_loss_cfg = OmegaConf.to_container(config.loss.edge)
|
|
edge_loss_cfg.pop("weight")
|
|
edge_loss = EdgeLoss(**edge_loss_cfg).to(idist.device())
|
|
|
|
perceptual_loss_cfg = OmegaConf.to_container(config.loss.perceptual)
|
|
perceptual_loss_cfg.pop("weight")
|
|
perceptual_loss = PerceptualLoss(**perceptual_loss_cfg).to(idist.device())
|
|
|
|
recon_loss = nn.L1Loss() if config.loss.recon.level == 1 else nn.MSELoss()
|
|
|
|
image_buffers = {k: GANImageBuffer(config.data.train.buffer_size or 50) for k in discriminators.keys()}
|
|
|
|
def _step(engine, batch):
|
|
batch = convert_tensor(batch, idist.device())
|
|
real = dict(a=batch["a"], b=batch["b"])
|
|
fake = dict(
|
|
a=generator(content_img=batch["edge_a"], style_img=real["a"], which_decoder="a"),
|
|
b=generator(content_img=batch["edge_a"], style_img=real["b"], which_decoder="b"),
|
|
)
|
|
rec_b = generator(content_img=batch["edge_b"], style_img=real["b"], which_decoder="b")
|
|
rec_bb = generator(content_img=batch["edge_b"], style_img=fake["b"], which_decoder="b")
|
|
|
|
optimizers["g"].zero_grad()
|
|
loss_g = dict()
|
|
for d in "ab":
|
|
discriminators[d].requires_grad_(False)
|
|
pred_fake = discriminators[d](fake[d])
|
|
loss_g[f"gan_{d}"] = config.loss.gan.weight * gan_loss(pred_fake, True)
|
|
_, t = perceptual_loss(fake[d], real[d])
|
|
loss_g[f"perceptual_{d}"] = config.loss.perceptual.weight * t
|
|
loss_g[f"edge_{d}"] = config.loss.edge.weight * edge_loss(fake[d], batch["edge_a"])
|
|
loss_g["recon_a"] = config.loss.recon.weight * recon_loss(fake["a"], real["a"])
|
|
loss_g["recon_b"] = config.loss.recon.weight * recon_loss(rec_b, real["b"])
|
|
loss_g["recon_bb"] = config.loss.recon.weight * recon_loss(rec_bb, real["b"])
|
|
sum(loss_g.values()).backward()
|
|
optimizers["g"].step()
|
|
|
|
for discriminator in discriminators.values():
|
|
discriminator.requires_grad_(True)
|
|
|
|
optimizers["d"].zero_grad()
|
|
loss_d = dict()
|
|
for k in discriminators.keys():
|
|
pred_real = discriminators[k](real[k])
|
|
pred_fake = discriminators[k](image_buffers[k].query(fake[k].detach()))
|
|
loss_d[f"gan_{k}"] = (gan_loss(pred_real, True, is_discriminator=True) +
|
|
gan_loss(pred_fake, False, is_discriminator=True)) / 2
|
|
sum(loss_d.values()).backward()
|
|
optimizers["d"].step()
|
|
|
|
generated_img = {f"real_{k}": real[k].detach() for k in real}
|
|
generated_img["rec_b"] = rec_b.detach()
|
|
generated_img["rec_bb"] = rec_b.detach()
|
|
generated_img.update({f"fake_{k}": fake[k].detach() for k in fake})
|
|
generated_img.update({f"edge_{k}": batch[f"edge_{k}"].expand(-1, 3, -1, -1).detach() for k in "ab"})
|
|
return {
|
|
"loss": {
|
|
"g": {ln: loss_g[ln].mean().item() for ln in loss_g},
|
|
"d": {ln: loss_d[ln].mean().item() for ln in loss_d},
|
|
},
|
|
"img": generated_img
|
|
}
|
|
|
|
trainer = Engine(_step)
|
|
trainer.logger = logger
|
|
for lr_shd in lr_schedulers.values():
|
|
trainer.add_event_handler(Events.ITERATION_COMPLETED, lr_shd)
|
|
|
|
RunningAverage(output_transform=lambda x: sum(x["loss"]["g"].values())).attach(trainer, "loss_g")
|
|
RunningAverage(output_transform=lambda x: sum(x["loss"]["d"].values())).attach(trainer, "loss_d")
|
|
|
|
to_save = dict(trainer=trainer)
|
|
to_save.update({f"lr_scheduler_{k}": lr_schedulers[k] for k in lr_schedulers})
|
|
to_save.update({f"optimizer_{k}": optimizers[k] for k in optimizers})
|
|
to_save.update({"generator": generator})
|
|
to_save.update({f"discriminator_{k}": discriminators[k] for k in discriminators})
|
|
setup_common_handlers(trainer, config, to_save=to_save, clear_cuda_cache=True, set_epoch_for_dist_sampler=True,
|
|
end_event=Events.ITERATION_COMPLETED(once=config.max_iteration))
|
|
|
|
def output_transform(output):
|
|
loss = dict()
|
|
for tl in output["loss"]:
|
|
if isinstance(output["loss"][tl], dict):
|
|
for l in output["loss"][tl]:
|
|
loss[f"{tl}_{l}"] = output["loss"][tl][l]
|
|
else:
|
|
loss[tl] = output["loss"][tl]
|
|
return loss
|
|
|
|
iter_per_epoch = len(train_data_loader)
|
|
tensorboard_handler = setup_tensorboard_handler(trainer, config, output_transform, iter_per_epoch)
|
|
if tensorboard_handler is not None:
|
|
tensorboard_handler.attach(
|
|
trainer,
|
|
log_handler=OptimizerParamsHandler(optimizers["g"], tag="optimizer_g"),
|
|
event_name=Events.ITERATION_STARTED(every=max(iter_per_epoch // config.interval.tensorboard.scalar, 1))
|
|
)
|
|
|
|
@trainer.on(Events.ITERATION_COMPLETED(every=max(iter_per_epoch // config.interval.tensorboard.image, 1)))
|
|
def show_images(engine):
|
|
output = engine.state.output
|
|
image_order = dict(
|
|
a=["edge_a", "real_a", "fake_a", "fake_b"],
|
|
b=["edge_b", "real_b", "rec_b", "rec_bb"]
|
|
)
|
|
for k in "ab":
|
|
tensorboard_handler.writer.add_image(
|
|
f"train/{k}",
|
|
make_2d_grid([output["img"][o] for o in image_order[k]]),
|
|
engine.state.iteration
|
|
)
|
|
|
|
with torch.no_grad():
|
|
g = torch.Generator()
|
|
g.manual_seed(config.misc.random_seed)
|
|
random_start = torch.randperm(len(engine.state.test_dataset) - 11, generator=g).tolist()[0]
|
|
test_images = dict(
|
|
a=[[], [], [], []],
|
|
b=[[], [], [], []]
|
|
)
|
|
for i in range(random_start, random_start + 10):
|
|
batch = convert_tensor(engine.state.test_dataset[i], idist.device())
|
|
for k in batch:
|
|
batch[k] = batch[k].view(1, *batch[k].size())
|
|
|
|
real = dict(a=batch["a"], b=batch["b"])
|
|
fake = dict(
|
|
a=generator(content_img=batch["edge_a"], style_img=real["a"], which_decoder="a"),
|
|
b=generator(content_img=batch["edge_a"], style_img=real["b"], which_decoder="b"),
|
|
)
|
|
rec_b = generator(content_img=batch["edge_b"], style_img=real["b"], which_decoder="b")
|
|
rec_bb = generator(content_img=batch["edge_b"], style_img=fake["b"], which_decoder="b")
|
|
|
|
test_images["a"][0].append(batch["edge_a"])
|
|
test_images["a"][1].append(batch["a"])
|
|
test_images["a"][2].append(fake["a"])
|
|
test_images["a"][3].append(fake["b"])
|
|
test_images["b"][0].append(batch["edge_b"])
|
|
test_images["b"][1].append(batch["b"])
|
|
test_images["b"][2].append(rec_b)
|
|
test_images["b"][3].append(rec_bb)
|
|
for n in "ab":
|
|
tensorboard_handler.writer.add_image(
|
|
f"test/{n}",
|
|
make_2d_grid([torch.cat(ti) for ti in test_images[n]]),
|
|
engine.state.iteration
|
|
)
|
|
|
|
return trainer
|
|
|
|
|
|
def run(task, config, logger):
|
|
assert torch.backends.cudnn.enabled
|
|
torch.backends.cudnn.benchmark = True
|
|
logger.info(f"start task {task}")
|
|
with read_write(config):
|
|
config.max_iteration = ceil(config.max_pairs / config.data.train.dataloader.batch_size)
|
|
|
|
if task == "train":
|
|
train_dataset = data.DATASET.build_with(config.data.train.dataset)
|
|
logger.info(f"train with dataset:\n{train_dataset}")
|
|
train_data_loader = idist.auto_dataloader(train_dataset, **config.data.train.dataloader)
|
|
trainer = get_trainer(config, logger, train_data_loader)
|
|
if idist.get_rank() == 0:
|
|
test_dataset = data.DATASET.build_with(config.data.test.dataset)
|
|
trainer.state.test_dataset = test_dataset
|
|
try:
|
|
trainer.run(train_data_loader, max_epochs=ceil(config.max_iteration / len(train_data_loader)))
|
|
except Exception:
|
|
import traceback
|
|
print(traceback.format_exc())
|
|
else:
|
|
return NotImplemented(f"invalid task: {task}")
|