configs/few-shot/crossdomain.yml

@@ -1,66 +0,0 @@
-name: cross-domain-1
-engine: crossdomain
-result_dir: ./result
-
-distributed:
-  model:
-    # broadcast_buffers: False
-
-misc:
-  random_seed: 1004
-
-checkpoints:
-  interval: 2000
-
-log:
-  logger:
-    level: 20 # DEBUG(10) INFO(20)
-
-model:
-  _type: resnet10
-
-baseline:
-  plusplus: False
-  optimizers:
-    _type: Adam
-  data:
-    dataloader:
-      batch_size: 1024
-      shuffle: True
-      num_workers: 16
-      pin_memory: True
-      drop_last: True
-    dataset:
-      train:
-        path: /data/few-shot/mini_imagenet_full_size/train
-        lmdb_path: /data/few-shot/lmdb/mini-ImageNet/train.lmdb
-        pipeline:
-          - Load
-          - RandomResizedCrop:
-              size: [256, 256]
-          - ColorJitter:
-              brightness: 0.4
-              contrast: 0.4
-              saturation: 0.4
-          - RandomHorizontalFlip
-          - ToTensor
-          - Normalize:
-              mean: [0.485, 0.456, 0.406]
-              std: [0.229, 0.224, 0.225]
-      val:
-        path: /data/few-shot/mini_imagenet_full_size/val
-        lmdb_path: /data/few-shot/lmdb/mini-ImageNet/val.lmdb
-        pipeline:
-          - Load
-          - Resize:
-              size: [286, 286]
-          - RandomCrop:
-              size: [256, 256]
-          - ToTensor
-          - Normalize:
-              mean: [0.485, 0.456, 0.406]
-              std: [0.229, 0.224, 0.225]
-
-
-
-

data/dataset.py

@@ -1,52 +1,23 @@
 import os
 import pickle
-from collections import defaultdict
 
 import torch
 from torch.utils.data import Dataset
-from torchvision.datasets import ImageFolder
+from torchvision.datasets.folder import has_file_allowed_extension, IMG_EXTENSIONS
-from torchvision.datasets.folder import has_file_allowed_extension, IMG_EXTENSIONS, default_loader
 
 import lmdb
-from tqdm import tqdm
 
 from .transform import transform_pipeline
 from .registry import DATASET
 
 
-def default_transform_way(transform, sample):
-    return [transform(sample[0]), *sample[1:]]
-
-
 class LMDBDataset(Dataset):
-    def __init__(self, lmdb_path, pipeline=None, transform_way=default_transform_way, map_size=2 ** 40, readonly=True,
+    def __init__(self, lmdb_path, output_transform=None, map_size=2 ** 40, readonly=True, **lmdb_kwargs):
-                 **lmdb_kwargs):
-        self.path = lmdb_path
         self.db = lmdb.open(lmdb_path, subdir=os.path.isdir(lmdb_path), map_size=map_size, readonly=readonly,
-                            lock=False, **lmdb_kwargs)
+                            **lmdb_kwargs)
-
+        self.output_transform = output_transform
         with self.db.begin(write=False) as txn:
-            self._len = pickle.loads(txn.get(b"$$len$$"))
+            self._len = pickle.loads(txn.get(b"__len__"))
-            self.done_pipeline = pickle.loads(txn.get(b"$$done_pipeline$$"))
-            if pipeline is None:
-                self.not_done_pipeline = []
-            else:
-                self.not_done_pipeline = self._remain_pipeline(pipeline)
-            self.transform = transform_pipeline(self.not_done_pipeline)
-            self.transform_way = transform_way
-            essential_attr = pickle.loads(txn.get(b"$$essential_attr$$"))
-            for ea in essential_attr:
-                setattr(self, ea, pickle.loads(txn.get(f"${ea}$".encode(encoding="utf-8"))))
-
-    def _remain_pipeline(self, pipeline):
-        for i, dp in enumerate(self.done_pipeline):
-            if pipeline[i] != dp:
-                raise ValueError(
-                    f"pipeline {self.done_pipeline} saved in this lmdb database is not match with pipeline:{pipeline}")
-        return pipeline[len(self.done_pipeline):]
-
-    def __repr__(self):
-        return f"LMDBDataset: {self.path}\nlength: {len(self)}\n{self.transform}"
 
     def __len__(self):
         return self._len
@@ -54,77 +25,10 @@ class LMDBDataset(Dataset):
     def __getitem__(self, idx):
         with self.db.begin(write=False) as txn:
             sample = pickle.loads(txn.get("{}".format(idx).encode()))
-            sample = self.transform_way(self.transform, sample)
+            if self.output_transform is not None:
+                sample = self.output_transform(sample)
             return sample
 
-    @staticmethod
-    def lmdbify(dataset, done_pipeline, lmdb_path):
-        env = lmdb.open(lmdb_path, map_size=2 ** 40, subdir=os.path.isdir(lmdb_path))
-        with env.begin(write=True) as txn:
-            for i in tqdm(range(len(dataset)), ncols=0):
-                txn.put("{}".format(i).encode(), pickle.dumps(dataset[i]))
-            txn.put(b"$$len$$", pickle.dumps(len(dataset)))
-            txn.put(b"$$done_pipeline$$", pickle.dumps(done_pipeline))
-            essential_attr = getattr(dataset, "essential_attr", list())
-            txn.put(b"$$essential_attr$$", pickle.dumps(essential_attr))
-            for ea in essential_attr:
-                txn.put(f"${ea}$".encode(encoding="utf-8"), pickle.dumps(getattr(dataset, ea)))
-
-
-@DATASET.register_module()
-class ImprovedImageFolder(ImageFolder):
-    def __init__(self, root, pipeline):
-        super().__init__(root, transform_pipeline(pipeline), loader=lambda x: x)
-        self.classes_list = defaultdict(list)
-        self.essential_attr = ["classes_list"]
-        for i in range(len(self)):
-            self.classes_list[self.samples[i][-1]].append(i)
-        assert len(self.classes_list) == len(self.classes)
-
-
-class EpisodicDataset(Dataset):
-    def __init__(self, origin_dataset, num_class, num_query, num_support, num_episodes):
-        self.origin = origin_dataset
-        self.num_class = num_class
-        assert self.num_class < len(self.origin.classes_list)
-        self.num_query = num_query  # K
-        self.num_support = num_support  # K
-        self.num_episodes = num_episodes
-
-    def _fetch_list_data(self, id_list):
-        return [self.origin[i][0] for i in id_list]
-
-    def __len__(self):
-        return self.num_episodes
-
-    def __getitem__(self, _):
-        random_classes = torch.randperm(len(self.origin.classes_list))[:self.num_class].tolist()
-        support_set_list = []
-        query_set_list = []
-        target_list = []
-        for tag, c in enumerate(random_classes):
-            image_list = self.origin.classes_list[c]
-
-            if len(image_list) >= self.num_query + self.num_support:
-                # have enough images belong to this class
-                idx_list = torch.randperm(len(image_list))[:self.num_query + self.num_support].tolist()
-            else:
-                idx_list = torch.randint(high=len(image_list), size=(self.num_query + self.num_support,)).tolist()
-
-            support = self._fetch_list_data(map(image_list.__getitem__, idx_list[:self.num_support]))
-            query = self._fetch_list_data(map(image_list.__getitem__, idx_list[self.num_support:]))
-            support_set_list.extend(support)
-            query_set_list.extend(query)
-            target_list.extend([tag] * self.num_query)
-        return {
-            "support": torch.stack(support_set_list),
-            "query": torch.stack(query_set_list),
-            "target": torch.tensor(target_list)
-        }
-
-    def __repr__(self):
-        return f"<EpisodicDataset NE{self.num_episodes} NC{self.num_class} NS{self.num_support} NQ{self.num_query}>"
-
 
 @DATASET.register_module()
 class SingleFolderDataset(Dataset):

engine/crossdomain.py

@@ -1,83 +0,0 @@
-import torch
-import torch.nn as nn
-from torchvision.datasets import ImageFolder
-
-import ignite.distributed as idist
-from ignite.contrib.metrics.gpu_info import GpuInfo
-from ignite.contrib.handlers.tensorboard_logger import TensorboardLogger, global_step_from_engine, OutputHandler, \
-    WeightsScalarHandler, GradsHistHandler, WeightsHistHandler, GradsScalarHandler
-from ignite.engine import create_supervised_evaluator, create_supervised_trainer, Events
-from ignite.metrics import Accuracy, Loss, RunningAverage
-from ignite.contrib.engines.common import save_best_model_by_val_score
-from ignite.contrib.handlers import ProgressBar
-
-from util.build import build_model, build_optimizer
-from util.handler import setup_common_handlers
-from data.transform import transform_pipeline
-from data.dataset import LMDBDataset
-
-
-def baseline_trainer(config, logger):
-    model = build_model(config.model, config.distributed.model)
-    optimizer = build_optimizer(model.parameters(), config.baseline.optimizers)
-    loss_fn = nn.CrossEntropyLoss()
-    trainer = create_supervised_trainer(model, optimizer, loss_fn, idist.device(), non_blocking=True,
-                                        output_transform=lambda x, y, y_pred, loss: (loss.item(), y_pred, y))
-    trainer.logger = logger
-    RunningAverage(output_transform=lambda x: x[0]).attach(trainer, "loss")
-    Accuracy(output_transform=lambda x: (x[1], x[2])).attach(trainer, "acc")
-    ProgressBar(ncols=0).attach(trainer)
-
-    if idist.get_rank() == 0:
-        GpuInfo().attach(trainer, name='gpu')
-
-        tb_logger = TensorboardLogger(log_dir=config.output_dir)
-        tb_logger.attach(
-            trainer,
-            log_handler=OutputHandler(
-                tag="train",
-                metric_names='all',
-                global_step_transform=global_step_from_engine(trainer),
-            ),
-            event_name=Events.EPOCH_COMPLETED
-        )
-        tb_logger.attach(trainer, log_handler=WeightsScalarHandler(model),
-                         event_name=Events.EPOCH_COMPLETED(every=10))
-
-        tb_logger.attach(trainer, log_handler=WeightsHistHandler(model), event_name=Events.EPOCH_COMPLETED(every=25))
-
-        tb_logger.attach(trainer, log_handler=GradsScalarHandler(model),
-                         event_name=Events.EPOCH_COMPLETED(every=10))
-
-        tb_logger.attach(trainer, log_handler=GradsHistHandler(model), event_name=Events.EPOCH_COMPLETED(every=25))
-
-        @trainer.on(Events.COMPLETED)
-        def _():
-            tb_logger.close()
-
-    to_save = dict(model=model, optimizer=optimizer, trainer=trainer)
-    setup_common_handlers(trainer, config.output_dir, print_interval_event=Events.EPOCH_COMPLETED, to_save=to_save,
-                          save_interval_event=Events.EPOCH_COMPLETED(every=25), n_saved=5,
-                          metrics_to_print=["loss", "acc"])
-    return trainer
-
-
-def run(task, config, logger):
-    assert torch.backends.cudnn.enabled
-    torch.backends.cudnn.benchmark = True
-    logger.info(f"start task {task}")
-    if task == "baseline":
-        train_dataset = LMDBDataset(config.baseline.data.dataset.train.lmdb_path,
-                                    pipeline=config.baseline.data.dataset.train.pipeline)
-        # train_dataset = ImageFolder(config.baseline.data.dataset.train.path,
-        #                             transform=transform_pipeline(config.baseline.data.dataset.train.pipeline))
-        logger.info(f"train with dataset:\n{train_dataset}")
-        train_data_loader = idist.auto_dataloader(train_dataset, **config.baseline.data.dataloader)
-        trainer = baseline_trainer(config, logger)
-        try:
-            trainer.run(train_data_loader, max_epochs=400)
-        except Exception:
-            import traceback
-            print(traceback.format_exc())
-    else:
-        return NotImplemented(f"invalid task: {task}")

main.py

@@ -30,10 +30,7 @@ def running(local_rank, config, task, backup_config=False, setup_output_dir=Fals
         output_dir = Path(config.result_dir) / config.name if config.output_dir is None else config.output_dir
         config.output_dir = str(output_dir)
         if output_dir.exists():
-            # assert not any(output_dir.iterdir()), "output_dir must be empty"
+            assert not any(output_dir.iterdir()), "output_dir must be empty"
-            contains = list(output_dir.iterdir())
-            assert (len(contains) == 0) or (len(contains) == 1 and contains[0].name == "config.yml"), \
-                f"output_dir must by empty or only contains config.yml, but now got {len(contains)} files"
         else:
             if idist.get_rank() == 0:
                 output_dir.mkdir(parents=True)

model/__init__.py

@@ -1,3 +1,2 @@
 from model.registry import MODEL
 import model.residual_generator
-import model.fewshot

model/fewshot.py

@@ -1,105 +0,0 @@
-import math
-
-import torch.nn as nn
-
-from .registry import MODEL
-
-
-# --- gaussian initialize ---
-def init_layer(l):
-    # Initialization using fan-in
-    if isinstance(l, nn.Conv2d):
-        n = l.kernel_size[0] * l.kernel_size[1] * l.out_channels
-        l.weight.data.normal_(0, math.sqrt(2.0 / float(n)))
-    elif isinstance(l, nn.BatchNorm2d):
-        l.weight.data.fill_(1)
-        l.bias.data.fill_(0)
-    elif isinstance(l, nn.Linear):
-        l.bias.data.fill_(0)
-
-
-class Flatten(nn.Module):
-    def __init__(self):
-        super(Flatten, self).__init__()
-
-    def forward(self, x):
-        return x.view(x.size(0), -1)
-
-
-class SimpleBlock(nn.Module):
-    def __init__(self, in_channels, out_channels, half_res, leakyrelu=False):
-        super(SimpleBlock, self).__init__()
-        self.in_channels = in_channels
-        self.out_channels = out_channels
-
-        self.block = nn.Sequential(
-            nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=2 if half_res else 1, padding=1, bias=False),
-            nn.BatchNorm2d(out_channels),
-            nn.ReLU(inplace=True) if not leakyrelu else nn.LeakyReLU(0.2, inplace=True),
-            nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1, bias=False),
-            nn.BatchNorm2d(out_channels),
-        )
-        self.relu = nn.ReLU(inplace=True) if not leakyrelu else nn.LeakyReLU(0.2, inplace=True)
-        if in_channels != out_channels:
-            self.shortcut = nn.Sequential(
-                nn.Conv2d(in_channels, out_channels, 1, 2 if half_res else 1, bias=False),
-                nn.BatchNorm2d(out_channels)
-            )
-        else:
-            self.shortcut = nn.Identity()
-
-    def forward(self, x):
-        o = self.block(x)
-        return self.relu(o + self.shortcut(x))
-
-
-class ResNet(nn.Module):
-    def __init__(self, block, layers, dims, num_classes=None, classifier_type="linear", flatten=True, leakyrelu=False):
-        super().__init__()
-        assert len(layers) == 4, 'Can have only four stages'
-        self.inplanes = 64
-
-        self.start = nn.Sequential(
-            nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False),
-            nn.BatchNorm2d(self.inplanes),
-            nn.ReLU(inplace=True) if not leakyrelu else nn.LeakyReLU(0.2, inplace=True),
-            nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
-        )
-
-        trunk = []
-        in_channels = self.inplanes
-        for i in range(4):
-            for j in range(layers[i]):
-                half_res = i >= 1 and j == 0
-                trunk.append(block(in_channels, dims[i], half_res, leakyrelu))
-                in_channels = dims[i]
-        if flatten:
-            trunk.append(nn.AvgPool2d(7))
-            trunk.append(Flatten())
-        if num_classes is not None:
-            if classifier_type == "linear":
-                trunk.append(nn.Linear(in_channels, num_classes))
-            elif classifier_type == "distlinear":
-                pass
-            else:
-                raise ValueError(f"invalid classifier_type:{classifier_type}")
-        self.trunk = nn.Sequential(*trunk)
-        self.apply(init_layer)
-
-    def forward(self, x):
-        return self.trunk(self.start(x))
-
-
-@MODEL.register_module()
-def resnet10(num_classes=None, classifier_type="linear", flatten=True, leakyrelu=False):
-    return ResNet(SimpleBlock, [1, 1, 1, 1], [64, 128, 256, 512], num_classes, classifier_type, flatten, leakyrelu)
-
-
-@MODEL.register_module()
-def resnet18(num_classes=None, classifier_type="linear", flatten=True, leakyrelu=False):
-    return ResNet(SimpleBlock, [2, 2, 2, 2], [64, 128, 256, 512], num_classes, classifier_type, flatten, leakyrelu)
-
-
-@MODEL.register_module()
-def resnet34(num_classes=None, classifier_type="linear", flatten=True, leakyrelu=False):
-    return ResNet(SimpleBlock, [3, 4, 6, 3], [64, 128, 256, 512], num_classes, classifier_type, flatten, leakyrelu)

run.sh

@@ -1,14 +1,8 @@
 #!/usr/bin/env bash
 
 CONFIG=$1
-TASK=$2
+GPUS=$2
-GPUS=$3
 
-_command="print(len('${GPUS}'.split(',')))"
+#  CUDA_VISIBLE_DEVICES=$GPUS \
-GPU_COUNT=$(python3 -c "${_command}")
+PYTHONPATH=.:$PYTHONPATH OMP_NUM_THREADS=1 python -m torch.distributed.launch --nproc_per_node="$GPUS" \
-
+  main.py train "$CONFIG" --backup_config --setup_output_dir --setup_random_seed
-echo "GPU_COUNT:${GPU_COUNT}"
-
-CUDA_VISIBLE_DEVICES=$GPUS \
-PYTHONPATH=.:$PYTHONPATH OMP_NUM_THREADS=1 python -m torch.distributed.launch --nproc_per_node="$GPU_COUNT" \
-  main.py "$TASK" "$CONFIG" --backup_config --setup_output_dir --setup_random_seed

tool/lmdbify.py

@@ -1,20 +0,0 @@
-import fire
-from omegaconf import OmegaConf
-from data.dataset import ImprovedImageFolder, LMDBDataset
-
-pipeline = """
-pipeline:
-    - Load
-"""
-
-
-def transform(dataset_path, save_path):
-    print(save_path, dataset_path)
-    conf = OmegaConf.create(pipeline)
-    print(conf.pipeline.pretty())
-    origin_dataset = ImprovedImageFolder(dataset_path, conf.pipeline)
-    LMDBDataset.lmdbify(origin_dataset, conf.pipeline, save_path)
-
-
-if __name__ == '__main__':
-    fire.Fire(transform)

util/handler.py

@@ -3,13 +3,13 @@ from pathlib import Path
 import torch
 
 import ignite.distributed as idist
-from ignite.engine import Events, Engine
+from ignite.engine import Events
 from ignite.handlers import Checkpoint, DiskSaver, TerminateOnNan
 from ignite.contrib.handlers import BasicTimeProfiler
 
 
 def setup_common_handlers(
-        trainer: Engine,
+        trainer,
         output_dir=None,
         stop_on_nan=True,
         use_profiler=True,
@@ -39,11 +39,6 @@ def setup_common_handlers(
     :param checkpoint_kwargs:
     :return:
     """
-    @trainer.on(Events.STARTED)
-    @idist.one_rank_only()
-    def print_dataloader_size(engine):
-        engine.logger.info(f"data loader length: {len(engine.state.dataloader)}")
-
     if stop_on_nan:
         trainer.add_event_handler(Events.ITERATION_COMPLETED, TerminateOnNan())
 
@@ -73,8 +68,6 @@ def setup_common_handlers(
         def print_interval(engine):
             print_str = f"epoch:{engine.state.epoch} iter:{engine.state.iteration}\t"
             for m in metrics_to_print:
-                if m not in engine.state.metrics:
-                    continue
                 print_str += f"{m}={engine.state.metrics[m]:.3f} "
             engine.logger.info(print_str)