From 3367f27c23bfb539848310d0d636302b44f7e4b8 Mon Sep 17 00:00:00 2001 From: lukeum Date: Tue, 20 Dec 2022 21:59:33 -0600 Subject: [PATCH 01/36] add texttext-clip --- src/open_clip/model.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/open_clip/model.py b/src/open_clip/model.py index 6b1abbf16..088d77e55 100644 --- a/src/open_clip/model.py +++ b/src/open_clip/model.py @@ -248,6 +248,47 @@ def forward(self, image, text): return image_features, text_features, self.logit_scale.exp() + +class TextTextCLIP(nn.Module): + def __init__( + self, + embed_dim: int, + query_cfg: CLIPTextCfg, + doc_cfg: CLIPTextCfg, + quick_gelu: bool = False, + cast_dtype: Optional[torch.dtype] = None, + ): + super().__init__() + self.doc = _build_text_tower(embed_dim, doc_cfg, quick_gelu, cast_dtype) + self.query = _build_text_tower(embed_dim, query_cfg, quick_gelu, cast_dtype) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + def lock_query_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): + self.query.lock(unlocked_layers, freeze_layer_norm) + + def lock_doc_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): + self.doc.lock(unlocked_layers, freeze_layer_norm) + + @torch.jit.ignore + def set_grad_checkpointing(self, enable=True): + self.doc.set_grad_checkpointing(enable) + self.query.set_grad_checkpointing(enable) + + def encode_doc(self, text, normalize: bool = False): + features = self.doc(text) + return F.normalize(features, dim=-1) if normalize else features + + def encode_query(self, text, normalize: bool = False): + features = self.query(text) + return F.normalize(features, dim=-1) if normalize else features + + def forward(self, query, doc): + query_features = self.encode_query(query, normalize=True) + doc_features = self.encode_doc(doc, normalize=True) + return query_features, doc_features, self.logit_scale.exp() + + + def convert_weights_to_lp(model: nn.Module, dtype=torch.float16): """Convert applicable model parameters to low-precision (bf16 or fp16)""" From 5089d57e8acb4ee89eed288ef9de56abc34f8747 Mon Sep 17 00:00:00 2001 From: lukeum Date: Wed, 21 Dec 2022 09:25:25 -0600 Subject: [PATCH 02/36] fix loss --- src/open_clip/__init__.py | 2 +- src/open_clip/loss.py | 72 +++++++++++++++++++-------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/open_clip/__init__.py b/src/open_clip/__init__.py index b76dd51b9..a15a917fd 100644 --- a/src/open_clip/__init__.py +++ b/src/open_clip/__init__.py @@ -2,7 +2,7 @@ from .factory import create_model, create_model_and_transforms, create_model_from_pretrained, get_tokenizer from .factory import list_models, add_model_config, get_model_config, load_checkpoint from .loss import ClipLoss -from .model import CLIP, CustomTextCLIP, CLIPTextCfg, CLIPVisionCfg,\ +from .model import CLIP, CustomTextCLIP, TextTextCLIP, CLIPTextCfg, CLIPVisionCfg,\ convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype from .openai import load_openai_model, list_openai_models from .pretrained import list_pretrained, list_pretrained_models_by_tag, list_pretrained_tags_by_model,\ diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py index de31426df..7f476cd85 100644 --- a/src/open_clip/loss.py +++ b/src/open_clip/loss.py @@ -16,8 +16,8 @@ def gather_features( - image_features, - text_features, + features_1, + features_2, local_loss=False, gather_with_grad=False, rank=0, @@ -28,38 +28,38 @@ def gather_features( if use_horovod: assert hvd is not None, 'Please install horovod' if gather_with_grad: - all_image_features = hvd.allgather(image_features) - all_text_features = hvd.allgather(text_features) + all_features_1 = hvd.allgather(features_1) + all_features_2 = hvd.allgather(features_2) else: with torch.no_grad(): - all_image_features = hvd.allgather(image_features) - all_text_features = hvd.allgather(text_features) + all_features_1 = hvd.allgather(features_1) + all_features_2 = hvd.allgather(features_2) if not local_loss: # ensure grads for local rank when all_* features don't have a gradient - gathered_image_features = list(all_image_features.chunk(world_size, dim=0)) - gathered_text_features = list(all_text_features.chunk(world_size, dim=0)) - gathered_image_features[rank] = image_features - gathered_text_features[rank] = text_features - all_image_features = torch.cat(gathered_image_features, dim=0) - all_text_features = torch.cat(gathered_text_features, dim=0) + gathered_features_1 = list(all_features_1.chunk(world_size, dim=0)) + gathered_features_2 = list(all_features_2.chunk(world_size, dim=0)) + gathered_features_1[rank] = features_1 + gathered_features_2[rank] = features_2 + all_features_1 = torch.cat(gathered_features_1, dim=0) + all_features_2 = torch.cat(gathered_features_2, dim=0) else: # We gather tensors from all gpus if gather_with_grad: - all_image_features = torch.cat(torch.distributed.nn.all_gather(image_features), dim=0) - all_text_features = torch.cat(torch.distributed.nn.all_gather(text_features), dim=0) + all_features_1 = torch.cat(torch.distributed.nn.all_gather(features_1), dim=0) + all_features_2 = torch.cat(torch.distributed.nn.all_gather(features_2), dim=0) else: - gathered_image_features = [torch.zeros_like(image_features) for _ in range(world_size)] - gathered_text_features = [torch.zeros_like(text_features) for _ in range(world_size)] - dist.all_gather(gathered_image_features, image_features) - dist.all_gather(gathered_text_features, text_features) + gathered_features_1 = [torch.zeros_like(features_1) for _ in range(world_size)] + gathered_features_2 = [torch.zeros_like(features_2) for _ in range(world_size)] + dist.all_gather(gathered_features_1, features_1) + dist.all_gather(gathered_features_2, features_2) if not local_loss: # ensure grads for local rank when all_* features don't have a gradient - gathered_image_features[rank] = image_features - gathered_text_features[rank] = text_features - all_image_features = torch.cat(gathered_image_features, dim=0) - all_text_features = torch.cat(gathered_text_features, dim=0) + gathered_features_1[rank] = features_1 + gathered_features_2[rank] = features_2 + all_features_1 = torch.cat(gathered_features_1, dim=0) + all_features_2 = torch.cat(gathered_features_2, dim=0) - return all_image_features, all_text_features + return all_features_1, all_features_2 class ClipLoss(nn.Module): @@ -85,25 +85,25 @@ def __init__( self.prev_num_logits = 0 self.labels = {} - def forward(self, image_features, text_features, logit_scale): - device = image_features.device + def forward(self, features_1, features_2, logit_scale): + device = features_1.device if self.world_size > 1: - all_image_features, all_text_features = gather_features( - image_features, text_features, + all_features_1, all_features_2 = gather_features( + features_1, features_2, self.local_loss, self.gather_with_grad, self.rank, self.world_size, self.use_horovod) if self.local_loss: - logits_per_image = logit_scale * image_features @ all_text_features.T - logits_per_text = logit_scale * text_features @ all_image_features.T + logits_per_feature_1 = logit_scale * features_1 @ all_features_2.T + logits_per_feature_2 = logit_scale * features_2 @ all_features_1.T else: - logits_per_image = logit_scale * all_image_features @ all_text_features.T - logits_per_text = logits_per_image.T + logits_per_feature_1 = logit_scale * all_features_1 @ all_features_2.T + logits_per_feature_2 = logits_per_feature_1.T else: - logits_per_image = logit_scale * image_features @ text_features.T - logits_per_text = logit_scale * text_features @ image_features.T + logits_per_feature_1 = logit_scale * features_1 @ features_2.T + logits_per_feature_2 = logit_scale * features_2 @ features_1.T # calculated ground-truth and cache if enabled - num_logits = logits_per_image.shape[0] + num_logits = logits_per_feature_1.shape[0] if self.prev_num_logits != num_logits or device not in self.labels: labels = torch.arange(num_logits, device=device, dtype=torch.long) if self.world_size > 1 and self.local_loss: @@ -115,7 +115,7 @@ def forward(self, image_features, text_features, logit_scale): labels = self.labels[device] total_loss = ( - F.cross_entropy(logits_per_image, labels) + - F.cross_entropy(logits_per_text, labels) + F.cross_entropy(logits_per_feature_1, labels) + + F.cross_entropy(logits_per_feature_2, labels) ) / 2 return total_loss From d414da58effe070fabd7b52bf17b72e2fe362685 Mon Sep 17 00:00:00 2001 From: lukeum Date: Mon, 26 Dec 2022 10:03:31 -0600 Subject: [PATCH 03/36] change main.py --- src/open_clip/factory.py | 84 ++++++++++++------- .../model_configs/roberta-roberta.json | 16 ++++ src/training/data.py | 51 +++++++++++ src/training/main.py | 8 ++ src/training/params.py | 43 +++++++++- 5 files changed, 169 insertions(+), 33 deletions(-) create mode 100644 src/open_clip/model_configs/roberta-roberta.json diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index bf07009bc..76e2ebcb3 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -10,7 +10,7 @@ import torch from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD -from .model import CLIP, CustomTextCLIP, convert_weights_to_lp, convert_to_custom_text_state_dict,\ +from .model import CLIP, CustomTextCLIP, TextTextCLIP, convert_weights_to_lp, convert_to_custom_text_state_dict,\ resize_pos_embed, get_cast_dtype from .openai import load_openai_model from .pretrained import is_pretrained_cfg, get_pretrained_cfg, download_pretrained, list_pretrained_tags_by_model @@ -109,6 +109,7 @@ def create_model( pretrained_image: bool = False, pretrained_hf: bool = True, cache_dir: Optional[str] = None, + text_to_text: Optional[bool] = False， ): model_name = model_name.replace('/', '-') # for callers using old naming with / in ViT names if isinstance(device, str): @@ -148,13 +149,22 @@ def create_model( cast_dtype = get_cast_dtype(precision) custom_text = model_cfg.pop('custom_text', False) or force_custom_text or ('hf_model_name' in model_cfg.get('text_cfg', {})) - - if custom_text: - if 'hf_model_name' in model_cfg.get('text_cfg', {}): - model_cfg['text_cfg']['hf_model_pretrained'] = pretrained_hf - model = CustomTextCLIP(**model_cfg, cast_dtype=cast_dtype) + + # switch to TextTextCLIP + if text_to_text: + if 'hf_model_name' in model_cfg.get('doc_cfg', {}): + model_cfg['doc_cfg']['hf_model_pretrained'] = pretrained_hf + if 'hf_model_name' in model_cfg.get('query_cfg', {}): + model_cfg['query_cfg']['hf_model_pretrained'] = pretrained_hf + + model = TextTextCLIP(**model_cfg, cast_dtype=cast_dtype) else: - model = CLIP(**model_cfg, cast_dtype=cast_dtype) + if custom_text: + if 'hf_model_name' in model_cfg.get('text_cfg', {}): + model_cfg['text_cfg']['hf_model_pretrained'] = pretrained_hf + model = CustomTextCLIP(**model_cfg, cast_dtype=cast_dtype) + else: + model = CLIP(**model_cfg, cast_dtype=cast_dtype) pretrained_cfg = {} if pretrained: @@ -179,9 +189,10 @@ def create_model( if precision in ("fp16", "bf16"): convert_weights_to_lp(model, dtype=torch.bfloat16 if precision == 'bf16' else torch.float16) + if not text_to_text: # set image / mean metadata from pretrained_cfg if available, or use default - model.visual.image_mean = pretrained_cfg.get('mean', None) or OPENAI_DATASET_MEAN - model.visual.image_std = pretrained_cfg.get('std', None) or OPENAI_DATASET_STD + model.visual.image_mean = pretrained_cfg.get('mean', None) or OPENAI_DATASET_MEAN + model.visual.image_std = pretrained_cfg.get('std', None) or OPENAI_DATASET_STD if jit: model = torch.jit.script(model) @@ -203,6 +214,7 @@ def create_model_and_transforms( image_mean: Optional[Tuple[float, ...]] = None, image_std: Optional[Tuple[float, ...]] = None, cache_dir: Optional[str] = None, + text_to_text: Optional[bool] = False， ): model = create_model( model_name, @@ -216,23 +228,28 @@ def create_model_and_transforms( pretrained_image=pretrained_image, pretrained_hf=pretrained_hf, cache_dir=cache_dir, + text_to_text: Optional[bool] = False， ) - image_mean = image_mean or getattr(model.visual, 'image_mean', None) - image_std = image_std or getattr(model.visual, 'image_std', None) - preprocess_train = image_transform( - model.visual.image_size, - is_train=True, - mean=image_mean, - std=image_std - ) - preprocess_val = image_transform( - model.visual.image_size, - is_train=False, - mean=image_mean, - std=image_std - ) - + if not text_to_text: + image_mean = image_mean or getattr(model.visual, 'image_mean', None) + image_std = image_std or getattr(model.visual, 'image_std', None) + preprocess_train = image_transform( + model.visual.image_size, + is_train=True, + mean=image_mean, + std=image_std + ) + preprocess_val = image_transform( + model.visual.image_size, + is_train=False, + mean=image_mean, + std=image_std + ) + else: + preprocess_val = None + preprocess_train = None + return model, preprocess_train, preprocess_val @@ -248,6 +265,7 @@ def create_model_from_pretrained( image_mean: Optional[Tuple[float, ...]] = None, image_std: Optional[Tuple[float, ...]] = None, cache_dir: Optional[str] = None, + text_to_text: Optional[bool] = False, ): if not is_pretrained_cfg(model_name, pretrained) and not os.path.exists(pretrained): raise RuntimeError( @@ -263,18 +281,20 @@ def create_model_from_pretrained( force_quick_gelu=force_quick_gelu, force_custom_text=force_custom_text, cache_dir=cache_dir, + text_to_text=text_to_text, ) if not return_transform: return model - image_mean = image_mean or getattr(model.visual, 'image_mean', None) - image_std = image_std or getattr(model.visual, 'image_std', None) - preprocess = image_transform( - model.visual.image_size, - is_train=False, - mean=image_mean, - std=image_std - ) + if not text_to_text: + image_mean = image_mean or getattr(model.visual, 'image_mean', None) + image_std = image_std or getattr(model.visual, 'image_std', None) + preprocess = image_transform( + model.visual.image_size, + is_train=False, + mean=image_mean, + std=image_std + ) return model, preprocess diff --git a/src/open_clip/model_configs/roberta-roberta.json b/src/open_clip/model_configs/roberta-roberta.json new file mode 100644 index 000000000..ea745cc47 --- /dev/null +++ b/src/open_clip/model_configs/roberta-roberta.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "query_cfg": { + "hf_model_name": "roberta-base", + "hf_tokenizer_name": "roberta-base", + "proj": "mlp", + "pooler_type": "mean_pooler" + }, + "doc_cfg": { + "hf_model_name": "roberta-base", + "hf_tokenizer_name": "roberta-base", + "proj": "mlp", + "pooler_type": "mean_pooler" + } +} \ No newline at end of file diff --git a/src/training/data.py b/src/training/data.py index e0b15a156..4cdf6f2cc 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -27,6 +27,55 @@ hvd = None +class TextPairDataset(Dataset): + def __init__(self, input_filename, query_key, doc_key, sep="\t", tokenizer=None): + logging.debug(f'Loading csv data from {input_filename}.') + df = pd.read_csv(input_filename, sep=sep) + + self.docs = df[doc_key].tolist() + self.queries = df[query_key].tolist() + logging.debug('Done loading data.') + + self.tokenize = tokenizer + + def __len__(self): + return len(self.docs) + + def __getitem__(self, idx): + docs = self.tokenize([str(self.docs[idx])])[0] + queries = self.tokenize([str(self.queries[idx])])[0] + return docs, queries + + +def get_text_pair_dataset(args, is_train, epoch=0, tokenizer=None): + input_filename = args.train_data if is_train else args.val_data + assert input_filename + dataset = CsvDataset( + input_filename, + query_key=args.csv_query_key, + doc_key=args.csv_doc_key, + sep=args.csv_separator, + tokenizer=tokenizer + ) + num_samples = len(dataset) + sampler = DistributedSampler(dataset) if args.distributed and is_train else None + shuffle = is_train and sampler is None + + dataloader = DataLoader( + dataset, + batch_size=args.batch_size, + shuffle=shuffle, + num_workers=args.workers, + pin_memory=True, + sampler=sampler, + drop_last=is_train, + ) + dataloader.num_samples = num_samples + dataloader.num_batches = len(dataloader) + + return DataInfo(dataloader, sampler) + + class CsvDataset(Dataset): def __init__(self, input_filename, transforms, img_key, caption_key, sep="\t", tokenizer=None): logging.debug(f'Loading csv data from {input_filename}.') @@ -475,6 +524,8 @@ def get_synthetic_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None def get_dataset_fn(data_path, dataset_type): + if dataset_type == 'textpair': + return get_text_pair_dataset if dataset_type == "webdataset": return get_wds_dataset elif dataset_type == "csv": diff --git a/src/training/main.py b/src/training/main.py index 26d4cc529..5ae1dcd5a 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -146,6 +146,14 @@ def main(args): model.lock_text_tower( unlocked_layers=args.lock_text_unlocked_layers, freeze_layer_norm=args.lock_text_freeze_layer_norm) + if args.lock_doc: + model.lock_doc_tower( + unlocked_layers=args.lock_doc_unlocked_layers, + freeze_layer_norm=args.lock_doc_freeze_layer_norm) + if args.lock_text: + model.lock_query_tower( + unlocked_layers=args.lock_query_unlocked_layers, + freeze_layer_norm=args.lock_query_freeze_layer_norm) if args.grad_checkpointing: model.set_grad_checkpointing() diff --git a/src/training/params.py b/src/training/params.py index abc07dd50..2c5159bab 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -326,13 +326,54 @@ def parse_args(args): action='store_true', help="Freeze BatchNorm running stats in image tower for any locked layers.", ) + parser.add_argument( + "--lock-doc", + default=False, + action='store_true', + help="Lock full text tower by disabling gradients.", + ) + parser.add_argument( + "--lock-doc-unlocked-layers", + type=int, + default=0, + help="Leave last n image tower layer groups unlocked.", + ) + parser.add_argument( + "--lock-doc-freeze-layer-norm", + default=False, + action='store_true', + help="Freeze BatchNorm running stats in image tower for any locked layers.", + ) + parser.add_argument( + "--lock-query", + default=False, + action='store_true', + help="Lock full text tower by disabling gradients.", + ) + parser.add_argument( + "--lock-query-unlocked-layers", + type=int, + default=0, + help="Leave last n image tower layer groups unlocked.", + ) + parser.add_argument( + "--lock-query-freeze-layer-norm", + default=False, + action='store_true', + help="Freeze BatchNorm running stats in image tower for any locked layers.", + ) parser.add_argument( "--log-every-n-steps", type=int, default=100, help="Log every n steps to tensorboard/console/wandb.", ) - + parser.add_argument( + "--text-to-text", + type=bool, + default=False, + help="Train a TextTextCLIP model.", + ) args = parser.parse_args(args) From 949834e665e5d690e9bf9adc1200e015a92e673b Mon Sep 17 00:00:00 2001 From: lukeum Date: Mon, 26 Dec 2022 10:17:26 -0600 Subject: [PATCH 04/36] add arguments --- src/open_clip/factory.py | 4 ++-- src/training/main.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 76e2ebcb3..2433c27cc 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -228,7 +228,7 @@ def create_model_and_transforms( pretrained_image=pretrained_image, pretrained_hf=pretrained_hf, cache_dir=cache_dir, - text_to_text: Optional[bool] = False， + text_to_text， ) if not text_to_text: @@ -249,7 +249,7 @@ def create_model_and_transforms( else: preprocess_val = None preprocess_train = None - + return model, preprocess_train, preprocess_val diff --git a/src/training/main.py b/src/training/main.py index 5ae1dcd5a..07afac710 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -131,6 +131,7 @@ def main(args): pretrained_image=args.pretrained_image, image_mean=args.image_mean, image_std=args.image_std, + text_to_text=args.text_to_text, ) random_seed(args.seed, args.rank) From 479aa07fa63575ad1f594c9a0a23ff5bbed1c2df Mon Sep 17 00:00:00 2001 From: lukeum Date: Tue, 27 Dec 2022 15:59:17 -0600 Subject: [PATCH 05/36] add factory.py test --- src/open_clip/factory.py | 14 +++++++++----- src/training/data.py | 2 +- tests/test_TextTextCLIP.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 tests/test_TextTextCLIP.py diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 2433c27cc..168858aeb 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -41,7 +41,7 @@ def _rescan_model_configs(): for cf in config_files: with open(cf, 'r') as f: model_cfg = json.load(f) - if all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')): + if all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')) or all(a in model_cfg for a in ('embed_dim', 'query_cfg', 'doc_cfg')): _MODEL_CONFIGS[cf.stem] = model_cfg _MODEL_CONFIGS = {k: v for k, v in sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0]))} @@ -72,7 +72,11 @@ def get_model_config(model_name): def get_tokenizer(model_name): config = get_model_config(model_name) - tokenizer = HFTokenizer(config['text_cfg']['hf_tokenizer_name']) if 'hf_tokenizer_name' in config['text_cfg'] else tokenize + if 'text_cfg' in config.keys(): + key = 'text_cfg' + elif 'query_cfg' in config.keys(): + key = 'query_cfg' + tokenizer = HFTokenizer(config[key]['hf_tokenizer_name']) if 'hf_tokenizer_name' in config[key] else tokenize return tokenizer @@ -109,7 +113,7 @@ def create_model( pretrained_image: bool = False, pretrained_hf: bool = True, cache_dir: Optional[str] = None, - text_to_text: Optional[bool] = False， + text_to_text: Optional[bool] = False, ): model_name = model_name.replace('/', '-') # for callers using old naming with / in ViT names if isinstance(device, str): @@ -214,7 +218,7 @@ def create_model_and_transforms( image_mean: Optional[Tuple[float, ...]] = None, image_std: Optional[Tuple[float, ...]] = None, cache_dir: Optional[str] = None, - text_to_text: Optional[bool] = False， + text_to_text: Optional[bool] = False, ): model = create_model( model_name, @@ -228,7 +232,7 @@ def create_model_and_transforms( pretrained_image=pretrained_image, pretrained_hf=pretrained_hf, cache_dir=cache_dir, - text_to_text， + text_to_text=text_to_text, ) if not text_to_text: diff --git a/src/training/data.py b/src/training/data.py index 4cdf6f2cc..72f3230ba 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -47,7 +47,7 @@ def __getitem__(self, idx): return docs, queries -def get_text_pair_dataset(args, is_train, epoch=0, tokenizer=None): +def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None): input_filename = args.train_data if is_train else args.val_data assert input_filename dataset = CsvDataset( diff --git a/tests/test_TextTextCLIP.py b/tests/test_TextTextCLIP.py new file mode 100644 index 000000000..93a9da17a --- /dev/null +++ b/tests/test_TextTextCLIP.py @@ -0,0 +1,29 @@ +import torch +from open_clip.factory import get_tokenizer +import pytest +import open_clip +import os +os.environ["CUDA_VISIBLE_DEVICES"] = "" + +@pytest.mark.parametrize("model_type", [("roberta-roberta")]) +def test_inference_simple(model_type, pretrained): + model, _, preprocess = open_clip.create_model_and_transforms(model_type, pretrained=pretrained, jit=False, text_to_text=True) + tokenizer = get_tokenizer(model_type) + + doc = tokenizer(['this', 'is', 'a', 'document']) + query = tokenizer(['this', 'is', 'a', 'summary']) + + with torch.no_grad(): + doc_features = model.encode_doc(doc) + query_features = model.encode_query(query) + + print(doc_features.shape) + print(query_features.shape) + + text_probs = (100.0 * doc_features @ query_features.T).softmax(dim=-1) + print(text_probs) + +if __name__ == "__main__": + #open_clip.factory._rescan_model_configs() + print(open_clip.factory._MODEL_CONFIG_PATHS) + test_inference_simple("roberta-roberta",'') \ No newline at end of file From 8fcc3aadee2a14065d7e676cc000d73b6f00595c Mon Sep 17 00:00:00 2001 From: lukeum Date: Tue, 3 Jan 2023 21:52:17 -0600 Subject: [PATCH 06/36] test main.py --- src/training/data.py | 9 ++++----- src/training/params.py | 14 +++++++++++++- tests/test_TextTextCLIP.py | 32 +++++++++++++++++++++++++++++--- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/src/training/data.py b/src/training/data.py index 72f3230ba..efb6c5991 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -28,9 +28,9 @@ class TextPairDataset(Dataset): - def __init__(self, input_filename, query_key, doc_key, sep="\t", tokenizer=None): - logging.debug(f'Loading csv data from {input_filename}.') - df = pd.read_csv(input_filename, sep=sep) + def __init__(self, input_filename, query_key, doc_key, tokenizer=None): + logging.debug(f'Loading parquet data from {input_filename}.') + df = pd.read_parquet(input_filename) self.docs = df[doc_key].tolist() self.queries = df[query_key].tolist() @@ -50,11 +50,10 @@ def __getitem__(self, idx): def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None): input_filename = args.train_data if is_train else args.val_data assert input_filename - dataset = CsvDataset( + dataset = TextPairDataset( input_filename, query_key=args.csv_query_key, doc_key=args.csv_doc_key, - sep=args.csv_separator, tokenizer=tokenizer ) num_samples = len(dataset) diff --git a/src/training/params.py b/src/training/params.py index 2c5159bab..2bda83803 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -38,7 +38,7 @@ def parse_args(args): ) parser.add_argument( "--dataset-type", - choices=["webdataset", "csv", "synthetic", "auto"], + choices=["webdataset", "csv", "synthetic", "auto", "textpair"], default="auto", help="Which type of dataset to process." ) @@ -66,6 +66,18 @@ def parse_args(args): default="title", help="For csv-like datasets, the name of the key for the captions." ) + parser.add_argument( + "--csv-query-key", + type=str, + default="query", + help="For csv-like datasets, the name of the key for the image paths." + ) + parser.add_argument( + "--csv-doc-key", + type=str, + default="doc", + help="For csv-like datasets, the name of the key for the captions." + ) parser.add_argument( "--imagenet-val", type=str, diff --git a/tests/test_TextTextCLIP.py b/tests/test_TextTextCLIP.py index 93a9da17a..3355f00f5 100644 --- a/tests/test_TextTextCLIP.py +++ b/tests/test_TextTextCLIP.py @@ -3,10 +3,16 @@ import pytest import open_clip import os +import sys +import pandas as pd +from training.data import get_data +from training.params import parse_args +from training.main import main + os.environ["CUDA_VISIBLE_DEVICES"] = "" @pytest.mark.parametrize("model_type", [("roberta-roberta")]) -def test_inference_simple(model_type, pretrained): +def test_inference_simple(model_type, pretrained=None): model, _, preprocess = open_clip.create_model_and_transforms(model_type, pretrained=pretrained, jit=False, text_to_text=True) tokenizer = get_tokenizer(model_type) @@ -23,7 +29,27 @@ def test_inference_simple(model_type, pretrained): text_probs = (100.0 * doc_features @ query_features.T).softmax(dim=-1) print(text_probs) + ''' if __name__ == "__main__": + #open_clip.factory._rescan_model_configs() - print(open_clip.factory._MODEL_CONFIG_PATHS) - test_inference_simple("roberta-roberta",'') \ No newline at end of file + #print(open_clip.factory._MODEL_CONFIG_PATHS) + #test_inference_simple("roberta-roberta",'') + + main([ + '--save-frequency', '1', + '--zeroshot-frequency', '1', + '--dataset-type', "synthetic", + '--train-num-samples', '16', + '--warmup', '1', + '--batch-size', '2', + '--lr', '1e-3', + '--wd', '0.1', + '--epochs', '1', + '--workers', '2', + '--model', 'roberta-roberta', + '--dataset-type', 'textpair', + '--train-data', 'text_pairs.parquet.gzip', + '--text-to-text', 'True', + ]) + ''' \ No newline at end of file From e68f4f84820f0274cc56bc3ef28cd28b2e2aea1e Mon Sep 17 00:00:00 2001 From: lukeum Date: Wed, 4 Jan 2023 20:24:39 -0600 Subject: [PATCH 07/36] fix main.py --- src/training/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/training/main.py b/src/training/main.py index 07afac710..89c5772b8 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -151,7 +151,7 @@ def main(args): model.lock_doc_tower( unlocked_layers=args.lock_doc_unlocked_layers, freeze_layer_norm=args.lock_doc_freeze_layer_norm) - if args.lock_text: + if args.lock_query: model.lock_query_tower( unlocked_layers=args.lock_query_unlocked_layers, freeze_layer_norm=args.lock_query_freeze_layer_norm) From f6eba4f917a3eae91a56b9d51fc70b5d80066318 Mon Sep 17 00:00:00 2001 From: lukeum Date: Wed, 4 Jan 2023 21:30:01 -0600 Subject: [PATCH 08/36] rename variables --- src/open_clip/factory.py | 14 ++-- src/open_clip/loss.py | 72 +++++++++---------- src/open_clip/model.py | 36 +++++----- .../model_configs/roberta-roberta.json | 4 +- src/training/data.py | 18 ++--- src/training/main.py | 16 ++--- src/training/params.py | 24 +++---- tests/test_TextTextCLIP.py | 8 +-- 8 files changed, 96 insertions(+), 96 deletions(-) diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 168858aeb..72e6d28a0 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -41,7 +41,7 @@ def _rescan_model_configs(): for cf in config_files: with open(cf, 'r') as f: model_cfg = json.load(f) - if all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')) or all(a in model_cfg for a in ('embed_dim', 'query_cfg', 'doc_cfg')): + if all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')) or all(a in model_cfg for a in ('embed_dim', 'tower_a_cfg', 'tower_b_cfg')): _MODEL_CONFIGS[cf.stem] = model_cfg _MODEL_CONFIGS = {k: v for k, v in sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0]))} @@ -74,8 +74,8 @@ def get_tokenizer(model_name): config = get_model_config(model_name) if 'text_cfg' in config.keys(): key = 'text_cfg' - elif 'query_cfg' in config.keys(): - key = 'query_cfg' + elif 'tower_a_cfg' in config.keys(): + key = 'tower_a_cfg' tokenizer = HFTokenizer(config[key]['hf_tokenizer_name']) if 'hf_tokenizer_name' in config[key] else tokenize return tokenizer @@ -156,10 +156,10 @@ def create_model( # switch to TextTextCLIP if text_to_text: - if 'hf_model_name' in model_cfg.get('doc_cfg', {}): - model_cfg['doc_cfg']['hf_model_pretrained'] = pretrained_hf - if 'hf_model_name' in model_cfg.get('query_cfg', {}): - model_cfg['query_cfg']['hf_model_pretrained'] = pretrained_hf + if 'hf_model_name' in model_cfg.get('tower_a_cfg', {}): + model_cfg['tower_a_cfg']['hf_model_pretrained'] = pretrained_hf + if 'hf_model_name' in model_cfg.get('tower_b_cfg', {}): + model_cfg['tower_b_cfg']['hf_model_pretrained'] = pretrained_hf model = TextTextCLIP(**model_cfg, cast_dtype=cast_dtype) else: diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py index 7f476cd85..64e70463e 100644 --- a/src/open_clip/loss.py +++ b/src/open_clip/loss.py @@ -16,8 +16,8 @@ def gather_features( - features_1, - features_2, + features_a, + features_b, local_loss=False, gather_with_grad=False, rank=0, @@ -28,38 +28,38 @@ def gather_features( if use_horovod: assert hvd is not None, 'Please install horovod' if gather_with_grad: - all_features_1 = hvd.allgather(features_1) - all_features_2 = hvd.allgather(features_2) + all_features_a = hvd.allgather(features_a) + all_features_b = hvd.allgather(features_b) else: with torch.no_grad(): - all_features_1 = hvd.allgather(features_1) - all_features_2 = hvd.allgather(features_2) + all_features_a = hvd.allgather(features_a) + all_features_b = hvd.allgather(features_b) if not local_loss: # ensure grads for local rank when all_* features don't have a gradient - gathered_features_1 = list(all_features_1.chunk(world_size, dim=0)) - gathered_features_2 = list(all_features_2.chunk(world_size, dim=0)) - gathered_features_1[rank] = features_1 - gathered_features_2[rank] = features_2 - all_features_1 = torch.cat(gathered_features_1, dim=0) - all_features_2 = torch.cat(gathered_features_2, dim=0) + gathered_features_a = list(all_features_a.chunk(world_size, dim=0)) + gathered_features_b = list(all_features_b.chunk(world_size, dim=0)) + gathered_features_a[rank] = features_a + gathered_features_b[rank] = features_b + all_features_a = torch.cat(gathered_features_a, dim=0) + all_features_b = torch.cat(gathered_features_b, dim=0) else: # We gather tensors from all gpus if gather_with_grad: - all_features_1 = torch.cat(torch.distributed.nn.all_gather(features_1), dim=0) - all_features_2 = torch.cat(torch.distributed.nn.all_gather(features_2), dim=0) + all_features_a = torch.cat(torch.distributed.nn.all_gather(features_a), dim=0) + all_features_b = torch.cat(torch.distributed.nn.all_gather(features_b), dim=0) else: - gathered_features_1 = [torch.zeros_like(features_1) for _ in range(world_size)] - gathered_features_2 = [torch.zeros_like(features_2) for _ in range(world_size)] - dist.all_gather(gathered_features_1, features_1) - dist.all_gather(gathered_features_2, features_2) + gathered_features_a = [torch.zeros_like(features_a) for _ in range(world_size)] + gathered_features_b = [torch.zeros_like(features_b) for _ in range(world_size)] + dist.all_gather(gathered_features_a, features_a) + dist.all_gather(gathered_features_b, features_b) if not local_loss: # ensure grads for local rank when all_* features don't have a gradient - gathered_features_1[rank] = features_1 - gathered_features_2[rank] = features_2 - all_features_1 = torch.cat(gathered_features_1, dim=0) - all_features_2 = torch.cat(gathered_features_2, dim=0) + gathered_features_a[rank] = features_a + gathered_features_b[rank] = features_b + all_features_a = torch.cat(gathered_features_a, dim=0) + all_features_b = torch.cat(gathered_features_b, dim=0) - return all_features_1, all_features_2 + return all_features_a, all_features_b class ClipLoss(nn.Module): @@ -85,25 +85,25 @@ def __init__( self.prev_num_logits = 0 self.labels = {} - def forward(self, features_1, features_2, logit_scale): - device = features_1.device + def forward(self, features_a, features_b, logit_scale): + device = features_a.device if self.world_size > 1: - all_features_1, all_features_2 = gather_features( - features_1, features_2, + all_features_a, all_features_b = gather_features( + features_a, features_b, self.local_loss, self.gather_with_grad, self.rank, self.world_size, self.use_horovod) if self.local_loss: - logits_per_feature_1 = logit_scale * features_1 @ all_features_2.T - logits_per_feature_2 = logit_scale * features_2 @ all_features_1.T + logits_per_feature_a = logit_scale * features_a @ all_features_b.T + logits_per_feature_b = logit_scale * features_b @ all_features_a.T else: - logits_per_feature_1 = logit_scale * all_features_1 @ all_features_2.T - logits_per_feature_2 = logits_per_feature_1.T + logits_per_feature_a = logit_scale * all_features_a @ all_features_b.T + logits_per_feature_b = logits_per_feature_a.T else: - logits_per_feature_1 = logit_scale * features_1 @ features_2.T - logits_per_feature_2 = logit_scale * features_2 @ features_1.T + logits_per_feature_a = logit_scale * features_a @ features_b.T + logits_per_feature_b = logit_scale * features_b @ features_a.T # calculated ground-truth and cache if enabled - num_logits = logits_per_feature_1.shape[0] + num_logits = logits_per_feature_a.shape[0] if self.prev_num_logits != num_logits or device not in self.labels: labels = torch.arange(num_logits, device=device, dtype=torch.long) if self.world_size > 1 and self.local_loss: @@ -115,7 +115,7 @@ def forward(self, features_1, features_2, logit_scale): labels = self.labels[device] total_loss = ( - F.cross_entropy(logits_per_feature_1, labels) + - F.cross_entropy(logits_per_feature_2, labels) + F.cross_entropy(logits_per_feature_a, labels) + + F.cross_entropy(logits_per_feature_b, labels) ) / 2 return total_loss diff --git a/src/open_clip/model.py b/src/open_clip/model.py index 088d77e55..14e9d7c78 100644 --- a/src/open_clip/model.py +++ b/src/open_clip/model.py @@ -253,39 +253,39 @@ class TextTextCLIP(nn.Module): def __init__( self, embed_dim: int, - query_cfg: CLIPTextCfg, - doc_cfg: CLIPTextCfg, + tower_a_cfg: CLIPTextCfg, + tower_b_cfg: CLIPTextCfg, quick_gelu: bool = False, cast_dtype: Optional[torch.dtype] = None, ): super().__init__() - self.doc = _build_text_tower(embed_dim, doc_cfg, quick_gelu, cast_dtype) - self.query = _build_text_tower(embed_dim, query_cfg, quick_gelu, cast_dtype) + self.tower_a = _build_text_tower(embed_dim, tower_a_cfg, quick_gelu, cast_dtype) + self.tower_b = _build_text_tower(embed_dim, tower_b_cfg, quick_gelu, cast_dtype) self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) - def lock_query_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): - self.query.lock(unlocked_layers, freeze_layer_norm) + def lock_tower_a(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): + self.tower_a.lock(unlocked_layers, freeze_layer_norm) - def lock_doc_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): - self.doc.lock(unlocked_layers, freeze_layer_norm) + def lock_tower_b(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): + self.tower_b.lock(unlocked_layers, freeze_layer_norm) @torch.jit.ignore def set_grad_checkpointing(self, enable=True): - self.doc.set_grad_checkpointing(enable) - self.query.set_grad_checkpointing(enable) + self.tower_a.set_grad_checkpointing(enable) + self.tower_b.set_grad_checkpointing(enable) - def encode_doc(self, text, normalize: bool = False): - features = self.doc(text) + def encode_text_a(self, text, normalize: bool = False): + features = self.tower_a(text) return F.normalize(features, dim=-1) if normalize else features - def encode_query(self, text, normalize: bool = False): - features = self.query(text) + def encode_text_b(self, text, normalize: bool = False): + features = self.tower_b(text) return F.normalize(features, dim=-1) if normalize else features - def forward(self, query, doc): - query_features = self.encode_query(query, normalize=True) - doc_features = self.encode_doc(doc, normalize=True) - return query_features, doc_features, self.logit_scale.exp() + def forward(self, text_a, text_b): + features_a = self.encode_text_a(text_a, normalize=True) + features_b = self.encode_text_b(text_b, normalize=True) + return features_a, features_b, self.logit_scale.exp() diff --git a/src/open_clip/model_configs/roberta-roberta.json b/src/open_clip/model_configs/roberta-roberta.json index ea745cc47..fe21c14df 100644 --- a/src/open_clip/model_configs/roberta-roberta.json +++ b/src/open_clip/model_configs/roberta-roberta.json @@ -1,13 +1,13 @@ { "embed_dim": 512, "quick_gelu": true, - "query_cfg": { + "tower_a_cfg": { "hf_model_name": "roberta-base", "hf_tokenizer_name": "roberta-base", "proj": "mlp", "pooler_type": "mean_pooler" }, - "doc_cfg": { + "tower_b_cfg": { "hf_model_name": "roberta-base", "hf_tokenizer_name": "roberta-base", "proj": "mlp", diff --git a/src/training/data.py b/src/training/data.py index c72a9edca..1830a3b4d 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -28,23 +28,23 @@ class TextPairDataset(Dataset): - def __init__(self, input_filename, query_key, doc_key, tokenizer=None): + def __init__(self, input_filename, text_a_key, text_b_key, tokenizer=None): logging.debug(f'Loading parquet data from {input_filename}.') df = pd.read_parquet(input_filename) - self.docs = df[doc_key].tolist() - self.queries = df[query_key].tolist() + self.text_a = df[text_a_key].tolist() + self.text_b = df[text_b_key].tolist() logging.debug('Done loading data.') self.tokenize = tokenizer def __len__(self): - return len(self.docs) + return len(self.text_a) def __getitem__(self, idx): - docs = self.tokenize([str(self.docs[idx])])[0] - queries = self.tokenize([str(self.queries[idx])])[0] - return docs, queries + texts_a = self.tokenize([str(self.text_a[idx])])[0] + texts_b = self.tokenize([str(self.text_b[idx])])[0] + return texts_a,texts_b def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None): @@ -52,8 +52,8 @@ def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None assert input_filename dataset = TextPairDataset( input_filename, - query_key=args.csv_query_key, - doc_key=args.csv_doc_key, + text_a_key=args.text_a_key, + text_b_key=args.text_b_key, tokenizer=tokenizer ) num_samples = len(dataset) diff --git a/src/training/main.py b/src/training/main.py index b8e0c9f95..ecc421e81 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -191,14 +191,14 @@ def main(args): model.lock_text_tower( unlocked_layers=args.lock_text_unlocked_layers, freeze_layer_norm=args.lock_text_freeze_layer_norm) - if args.lock_doc: - model.lock_doc_tower( - unlocked_layers=args.lock_doc_unlocked_layers, - freeze_layer_norm=args.lock_doc_freeze_layer_norm) - if args.lock_query: - model.lock_query_tower( - unlocked_layers=args.lock_query_unlocked_layers, - freeze_layer_norm=args.lock_query_freeze_layer_norm) + if args.lock_tower_a: + model.lock_tower_a( + unlocked_layers=args.lock_tower_a_unlocked_layers, + freeze_layer_norm=args.lock_tower_a_freeze_layer_norm) + if args.lock_tower_b: + model.lock_tower_b( + unlocked_layers=args.lock_tower_b_unlocked_layers, + freeze_layer_norm=args.lock_tower_b_freeze_layer_norm) if args.grad_checkpointing: model.set_grad_checkpointing() diff --git a/src/training/params.py b/src/training/params.py index 2bda83803..74cbb6e37 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -67,16 +67,16 @@ def parse_args(args): help="For csv-like datasets, the name of the key for the captions." ) parser.add_argument( - "--csv-query-key", + "--text-a-key", type=str, - default="query", - help="For csv-like datasets, the name of the key for the image paths." + default="text_a", + help="For text pair datasets, the name of the key for the texts a." ) parser.add_argument( - "--csv-doc-key", + "--text-b-key", type=str, - default="doc", - help="For csv-like datasets, the name of the key for the captions." + default="text_b", + help="For text pair datasets, the name of the key for the texts b." ) parser.add_argument( "--imagenet-val", @@ -339,37 +339,37 @@ def parse_args(args): help="Freeze BatchNorm running stats in image tower for any locked layers.", ) parser.add_argument( - "--lock-doc", + "--lock-tower-b", default=False, action='store_true', help="Lock full text tower by disabling gradients.", ) parser.add_argument( - "--lock-doc-unlocked-layers", + "--lock-tower-b-unlocked-layers", type=int, default=0, help="Leave last n image tower layer groups unlocked.", ) parser.add_argument( - "--lock-doc-freeze-layer-norm", + "--lock-tower-b-freeze-layer-norm", default=False, action='store_true', help="Freeze BatchNorm running stats in image tower for any locked layers.", ) parser.add_argument( - "--lock-query", + "--lock-tower-a", default=False, action='store_true', help="Lock full text tower by disabling gradients.", ) parser.add_argument( - "--lock-query-unlocked-layers", + "--lock-tower-a-unlocked-layers", type=int, default=0, help="Leave last n image tower layer groups unlocked.", ) parser.add_argument( - "--lock-query-freeze-layer-norm", + "--lock-tower-a-freeze-layer-norm", default=False, action='store_true', help="Freeze BatchNorm running stats in image tower for any locked layers.", diff --git a/tests/test_TextTextCLIP.py b/tests/test_TextTextCLIP.py index 3355f00f5..c5edfeca7 100644 --- a/tests/test_TextTextCLIP.py +++ b/tests/test_TextTextCLIP.py @@ -29,7 +29,7 @@ def test_inference_simple(model_type, pretrained=None): text_probs = (100.0 * doc_features @ query_features.T).softmax(dim=-1) print(text_probs) - ''' +''' if __name__ == "__main__": #open_clip.factory._rescan_model_configs() @@ -38,8 +38,6 @@ def test_inference_simple(model_type, pretrained=None): main([ '--save-frequency', '1', - '--zeroshot-frequency', '1', - '--dataset-type', "synthetic", '--train-num-samples', '16', '--warmup', '1', '--batch-size', '2', @@ -51,5 +49,7 @@ def test_inference_simple(model_type, pretrained=None): '--dataset-type', 'textpair', '--train-data', 'text_pairs.parquet.gzip', '--text-to-text', 'True', + '--text-a-key', 'query', + '--text-b-key', 'doc' ]) - ''' \ No newline at end of file +''' \ No newline at end of file From 5d331d972e85b59e7189720959b0a61b079e8c8d Mon Sep 17 00:00:00 2001 From: lukeum Date: Wed, 4 Jan 2023 21:37:54 -0600 Subject: [PATCH 09/36] rename variables --- tests/test_TextTextCLIP.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/test_TextTextCLIP.py b/tests/test_TextTextCLIP.py index c5edfeca7..dcd511d86 100644 --- a/tests/test_TextTextCLIP.py +++ b/tests/test_TextTextCLIP.py @@ -16,25 +16,22 @@ def test_inference_simple(model_type, pretrained=None): model, _, preprocess = open_clip.create_model_and_transforms(model_type, pretrained=pretrained, jit=False, text_to_text=True) tokenizer = get_tokenizer(model_type) - doc = tokenizer(['this', 'is', 'a', 'document']) - query = tokenizer(['this', 'is', 'a', 'summary']) + text_a = tokenizer(['this', 'is', 'a', 'document']) + text_b = tokenizer(['this', 'is', 'a', 'summary']) with torch.no_grad(): - doc_features = model.encode_doc(doc) - query_features = model.encode_query(query) + text_a_features = model.encode_text_a(text_a) + text_b_features = model.encode_text_b(text_b) - print(doc_features.shape) - print(query_features.shape) + print(text_a_features.shape) + print(text_b_features.shape) - text_probs = (100.0 * doc_features @ query_features.T).softmax(dim=-1) + text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) print(text_probs) ''' if __name__ == "__main__": - - #open_clip.factory._rescan_model_configs() - #print(open_clip.factory._MODEL_CONFIG_PATHS) - #test_inference_simple("roberta-roberta",'') + main([ '--save-frequency', '1', From 4a00ea0386298fb7ba4d189a5774ea06b46cd9fd Mon Sep 17 00:00:00 2001 From: lukeum Date: Mon, 23 Jan 2023 23:09:59 -0600 Subject: [PATCH 10/36] add hf datasets --- src/training/data.py | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/training/data.py b/src/training/data.py index 1bc67cc4a..23c2d8ddd 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -15,6 +15,7 @@ import torchvision.datasets as datasets import webdataset as wds from PIL import Image +from datasets import load_dataset from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler, IterableDataset, get_worker_info from torch.utils.data.distributed import DistributedSampler from webdataset.filters import _shuffle @@ -46,15 +47,44 @@ def __getitem__(self, idx): return texts_a,texts_b +class HFTextPairDataset(Dataset): + def __init__(self, input_filename, text_a_key, text_b_key, tokenizer=None): + logging.debug(f'Loading parquet data from {input_filename}.') + + self.dataset = load_dataset(input_filename) + self.text_a_key = text_a_key + self.text_b_key = text_b_key + + logging.debug('Done loading data.') + + self.tokenize = tokenizer + + def __len__(self): + return len(self.text_a) + + def __getitem__(self, idx): + texts_a = self.tokenize([str(self.dataset[self.text_a_key][idx])])[0] + texts_b = self.tokenize([str(self.dataset[self.text_b_key][idx])])[0] + return texts_a,texts_b + + def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None): input_filename = args.train_data if is_train else args.val_data assert input_filename - dataset = TextPairDataset( - input_filename, - text_a_key=args.text_a_key, - text_b_key=args.text_b_key, - tokenizer=tokenizer - ) + if input_filename.endswith('.parquet'): + dataset = TextPairDataset( + input_filename, + text_a_key=args.text_a_key, + text_b_key=args.text_b_key, + tokenizer=tokenizer + ) + else: + dataset = HFTextPairDataset( + input_filename, + text_a_key=args.text_a_key, + text_b_key=args.text_b_key, + tokenizer=tokenizer + ) num_samples = len(dataset) sampler = DistributedSampler(dataset) if args.distributed and is_train else None shuffle = is_train and sampler is None @@ -74,6 +104,8 @@ def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None return DataInfo(dataloader, sampler) + + class CsvDataset(Dataset): def __init__(self, input_filename, transforms, img_key, caption_key, sep="\t", tokenizer=None): logging.debug(f'Loading csv data from {input_filename}.') From 588e8baeccabd3420fe01abc6d388bb9f1946809 Mon Sep 17 00:00:00 2001 From: lukeum Date: Thu, 26 Jan 2023 15:46:50 -0600 Subject: [PATCH 11/36] fix Siamese network --- src/open_clip/__init__.py | 2 +- src/open_clip/factory.py | 29 ++++++++++------- src/open_clip/model.py | 31 +++++++++++++++++++ .../xlm-roberta-base-xlm-roberta-base.json | 16 ++++++++++ src/training/data.py | 1 + src/training/main.py | 2 +- src/training/params.py | 7 +++-- 7 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 src/open_clip/model_configs/xlm-roberta-base-xlm-roberta-base.json diff --git a/src/open_clip/__init__.py b/src/open_clip/__init__.py index a15a917fd..b2c0bdfb6 100644 --- a/src/open_clip/__init__.py +++ b/src/open_clip/__init__.py @@ -2,7 +2,7 @@ from .factory import create_model, create_model_and_transforms, create_model_from_pretrained, get_tokenizer from .factory import list_models, add_model_config, get_model_config, load_checkpoint from .loss import ClipLoss -from .model import CLIP, CustomTextCLIP, TextTextCLIP, CLIPTextCfg, CLIPVisionCfg,\ +from .model import CLIP, CustomTextCLIP, TextTextCLIP, SiameseTextCLIP, CLIPTextCfg, CLIPVisionCfg,\ convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype from .openai import load_openai_model, list_openai_models from .pretrained import list_pretrained, list_pretrained_models_by_tag, list_pretrained_tags_by_model,\ diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 72e6d28a0..592c8e090 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -10,7 +10,7 @@ import torch from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD -from .model import CLIP, CustomTextCLIP, TextTextCLIP, convert_weights_to_lp, convert_to_custom_text_state_dict,\ +from .model import CLIP, CustomTextCLIP, TextTextCLIP, SiameseTextCLIP, convert_weights_to_lp, convert_to_custom_text_state_dict,\ resize_pos_embed, get_cast_dtype from .openai import load_openai_model from .pretrained import is_pretrained_cfg, get_pretrained_cfg, download_pretrained, list_pretrained_tags_by_model @@ -113,7 +113,7 @@ def create_model( pretrained_image: bool = False, pretrained_hf: bool = True, cache_dir: Optional[str] = None, - text_to_text: Optional[bool] = False, + model_type: Optional[str] = "CLIP", ): model_name = model_name.replace('/', '-') # for callers using old naming with / in ViT names if isinstance(device, str): @@ -155,14 +155,19 @@ def create_model( custom_text = model_cfg.pop('custom_text', False) or force_custom_text or ('hf_model_name' in model_cfg.get('text_cfg', {})) # switch to TextTextCLIP - if text_to_text: + if model_type=="text_dual_encoder": if 'hf_model_name' in model_cfg.get('tower_a_cfg', {}): model_cfg['tower_a_cfg']['hf_model_pretrained'] = pretrained_hf if 'hf_model_name' in model_cfg.get('tower_b_cfg', {}): model_cfg['tower_b_cfg']['hf_model_pretrained'] = pretrained_hf - - model = TextTextCLIP(**model_cfg, cast_dtype=cast_dtype) - else: + model = TextTextCLIP(**model_cfg, cast_dtype=cast_dtype) + + elif model_type=="text_siamese_encoder": + if 'hf_model_name' in model_cfg.get('tower_a_cfg', {}): + model_cfg['tower_a_cfg']['hf_model_pretrained'] = pretrained_hf + model = SiameseTextCLIP(**model_cfg, cast_dtype=cast_dtype) + + elif model_type=="CLIP": if custom_text: if 'hf_model_name' in model_cfg.get('text_cfg', {}): model_cfg['text_cfg']['hf_model_pretrained'] = pretrained_hf @@ -218,7 +223,7 @@ def create_model_and_transforms( image_mean: Optional[Tuple[float, ...]] = None, image_std: Optional[Tuple[float, ...]] = None, cache_dir: Optional[str] = None, - text_to_text: Optional[bool] = False, + model_type: Optional[str] = "CLIP", ): model = create_model( model_name, @@ -232,10 +237,10 @@ def create_model_and_transforms( pretrained_image=pretrained_image, pretrained_hf=pretrained_hf, cache_dir=cache_dir, - text_to_text=text_to_text, + model_type=model_type, ) - if not text_to_text: + if model_type=="CLIP": image_mean = image_mean or getattr(model.visual, 'image_mean', None) image_std = image_std or getattr(model.visual, 'image_std', None) preprocess_train = image_transform( @@ -269,7 +274,7 @@ def create_model_from_pretrained( image_mean: Optional[Tuple[float, ...]] = None, image_std: Optional[Tuple[float, ...]] = None, cache_dir: Optional[str] = None, - text_to_text: Optional[bool] = False, + model_type: Optional[str] = "CLIP", ): if not is_pretrained_cfg(model_name, pretrained) and not os.path.exists(pretrained): raise RuntimeError( @@ -285,13 +290,13 @@ def create_model_from_pretrained( force_quick_gelu=force_quick_gelu, force_custom_text=force_custom_text, cache_dir=cache_dir, - text_to_text=text_to_text, + model_type=model_type, ) if not return_transform: return model - if not text_to_text: + if model_type=="CLIP": image_mean = image_mean or getattr(model.visual, 'image_mean', None) image_std = image_std or getattr(model.visual, 'image_std', None) preprocess = image_transform( diff --git a/src/open_clip/model.py b/src/open_clip/model.py index 14e9d7c78..db39f694b 100644 --- a/src/open_clip/model.py +++ b/src/open_clip/model.py @@ -289,6 +289,37 @@ def forward(self, text_a, text_b): + +class SiameseTextCLIP(nn.Module): + def __init__( + self, + embed_dim: int, + text_cfg: CLIPTextCfg, + quick_gelu: bool = False, + cast_dtype: Optional[torch.dtype] = None, + ): + super().__init__() + self.text_tower = _build_text_tower(embed_dim, text_cfg, quick_gelu, cast_dtype) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + def lock_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): + self.text_tower.lock(unlocked_layers, freeze_layer_norm) + + @torch.jit.ignore + def set_grad_checkpointing(self, enable=True): + self.text_tower.set_grad_checkpointing(enable) + + def encode_text(self, text, normalize: bool = False): + features = self.text_tower(text) + return F.normalize(features, dim=-1) if normalize else features + + def forward(self, text_a, text_b): + features_a = self.encode_text(text_a, normalize=True) + features_b = self.encode_text(text_b, normalize=True) + return features_a, features_b, self.logit_scale.exp() + + + def convert_weights_to_lp(model: nn.Module, dtype=torch.float16): """Convert applicable model parameters to low-precision (bf16 or fp16)""" diff --git a/src/open_clip/model_configs/xlm-roberta-base-xlm-roberta-base.json b/src/open_clip/model_configs/xlm-roberta-base-xlm-roberta-base.json new file mode 100644 index 000000000..766e71439 --- /dev/null +++ b/src/open_clip/model_configs/xlm-roberta-base-xlm-roberta-base.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "tower_a_cfg": { + "hf_model_name": "xlm-roberta-large", + "hf_tokenizer_name": "xlm-roberta-large", + "proj": "mlp", + "pooler_type": "mean_pooler" + }, + "tower_b_cfg": { + "hf_model_name": "xlm-roberta-large", + "hf_tokenizer_name": "xlm-roberta-large", + "proj": "mlp", + "pooler_type": "mean_pooler" + } +} \ No newline at end of file diff --git a/src/training/data.py b/src/training/data.py index 23c2d8ddd..371730093 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -85,6 +85,7 @@ def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None text_b_key=args.text_b_key, tokenizer=tokenizer ) + num_samples = len(dataset) sampler = DistributedSampler(dataset) if args.distributed and is_train else None shuffle = is_train and sampler is None diff --git a/src/training/main.py b/src/training/main.py index ecc421e81..0698117d5 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -175,7 +175,7 @@ def main(args): pretrained_image=args.pretrained_image, image_mean=args.image_mean, image_std=args.image_std, - text_to_text=args.text_to_text, + model_type=args.model_type, ) random_seed(args.seed, args.rank) diff --git a/src/training/params.py b/src/training/params.py index 74cbb6e37..ac354f471 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -381,10 +381,11 @@ def parse_args(args): help="Log every n steps to tensorboard/console/wandb.", ) parser.add_argument( - "--text-to-text", - type=bool, + "--model_type", + type=str, + choices=["CLIP", "text_dual_encoder", "text_siamese_encoder"], default=False, - help="Train a TextTextCLIP model.", + help="Specify the model type", ) args = parser.parse_args(args) From 1e0d6aaff619712df4dbdf5981542a5247926c5f Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Mon, 30 Jan 2023 15:42:15 -0600 Subject: [PATCH 12/36] fix some typos --- src/open_clip/factory.py | 2 +- src/open_clip/model.py | 2 +- ...berta-base.json => xlm-roberta-large-xlm-roberta-large.json} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/open_clip/model_configs/{xlm-roberta-base-xlm-roberta-base.json => xlm-roberta-large-xlm-roberta-large.json} (100%) diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 592c8e090..ba61f65fe 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -198,7 +198,7 @@ def create_model( if precision in ("fp16", "bf16"): convert_weights_to_lp(model, dtype=torch.bfloat16 if precision == 'bf16' else torch.float16) - if not text_to_text: + if model_type=="CLIP": # set image / mean metadata from pretrained_cfg if available, or use default model.visual.image_mean = pretrained_cfg.get('mean', None) or OPENAI_DATASET_MEAN model.visual.image_std = pretrained_cfg.get('std', None) or OPENAI_DATASET_STD diff --git a/src/open_clip/model.py b/src/open_clip/model.py index db39f694b..e0d645f16 100644 --- a/src/open_clip/model.py +++ b/src/open_clip/model.py @@ -302,7 +302,7 @@ def __init__( self.text_tower = _build_text_tower(embed_dim, text_cfg, quick_gelu, cast_dtype) self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) - def lock_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): + def lock_text_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): self.text_tower.lock(unlocked_layers, freeze_layer_norm) @torch.jit.ignore diff --git a/src/open_clip/model_configs/xlm-roberta-base-xlm-roberta-base.json b/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json similarity index 100% rename from src/open_clip/model_configs/xlm-roberta-base-xlm-roberta-base.json rename to src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json From 516176cfcc11c550801568bc6eefc1cf731abc13 Mon Sep 17 00:00:00 2001 From: lingjzhu <> Date: Mon, 30 Jan 2023 17:08:24 -0600 Subject: [PATCH 13/36] fix typos --- src/training/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/training/params.py b/src/training/params.py index ac354f471..e9d9c9d30 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -384,7 +384,7 @@ def parse_args(args): "--model_type", type=str, choices=["CLIP", "text_dual_encoder", "text_siamese_encoder"], - default=False, + default="CLIP", help="Specify the model type", ) From 71d46ea4c4d3079ebee342816dd58a17f493f95c Mon Sep 17 00:00:00 2001 From: lingjzhu <> Date: Mon, 30 Jan 2023 22:11:52 -0600 Subject: [PATCH 14/36] resolve conflicts --- models_gh_runner.txt | 0 src/open_clip/loss.py | 5 ++--- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 models_gh_runner.txt diff --git a/models_gh_runner.txt b/models_gh_runner.txt new file mode 100644 index 000000000..e69de29bb diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py index 19632e167..8eed22988 100644 --- a/src/open_clip/loss.py +++ b/src/open_clip/loss.py @@ -117,9 +117,8 @@ def forward(self, features_a, features_b, logit_scale,output_dict=False): else: labels = self.labels[device] - total_loss = ( - F.cross_entropy(logits_per_image, labels) + - F.cross_entropy(logits_per_text, labels) + total_loss = ( F.cross_entropy(logits_per_feature_a, labels) + + F.cross_entropy(logits_per_feature_b, labels) ) / 2 return {"contrastive_loss": total_loss} if output_dict else total_loss From 73ab4d787e230df7864749a3e61cdccd45d767a6 Mon Sep 17 00:00:00 2001 From: lingjzhu <> Date: Mon, 30 Jan 2023 22:24:06 -0600 Subject: [PATCH 15/36] resolve conflicts --- src/open_clip/factory.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index bde936474..dfa04ff8b 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -44,7 +44,11 @@ def _rescan_model_configs(): for cf in config_files: with open(cf, 'r') as f: model_cfg = json.load(f) - if all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')) or all(a in model_cfg for a in ('embed_dim', 'tower_a_cfg', 'tower_b_cfg')): + + is_clip = all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')) + is_text_only = all(a in model_cfg for a in ('embed_dim', 'tower_a_cfg', 'tower_b_cfg')) + is_multimodal = all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg', 'multimodal_cfg')) + if is_clip or is_text_only or is_multimodal: _MODEL_CONFIGS[cf.stem] = model_cfg _MODEL_CONFIGS = {k: v for k, v in sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0]))} From d2105349268c94f17b9f0ff9f7148a2bb18f732a Mon Sep 17 00:00:00 2001 From: lingjzhu <> Date: Mon, 30 Jan 2023 23:53:50 -0600 Subject: [PATCH 16/36] resolve conflicts --- src/open_clip/__init__.py | 1 - src/open_clip/factory.py | 1 + src/open_clip/loss.py | 12 ++++++++++-- tests/test_TextTextCLIP.py | 8 ++++---- tests/test_inference.py | 4 +++- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/open_clip/__init__.py b/src/open_clip/__init__.py index f24775cd3..e56db0266 100644 --- a/src/open_clip/__init__.py +++ b/src/open_clip/__init__.py @@ -4,7 +4,6 @@ from .loss import ClipLoss, CoCaLoss from .model import CLIP, CustomTextCLIP, TextTextCLIP, SiameseTextCLIP, CLIPTextCfg, CLIPVisionCfg,\ - convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype from .coca_model import CoCa from .openai import load_openai_model, list_openai_models diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index dfa04ff8b..5fe264e7d 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -52,6 +52,7 @@ def _rescan_model_configs(): _MODEL_CONFIGS[cf.stem] = model_cfg _MODEL_CONFIGS = {k: v for k, v in sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0]))} + print(_MODEL_CONFIGS) _rescan_model_configs() # initial populate of model config registry diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py index 8eed22988..c9d9fbb51 100644 --- a/src/open_clip/loss.py +++ b/src/open_clip/loss.py @@ -87,9 +87,17 @@ def __init__( self.labels = {} - def forward(self, features_a, features_b, logit_scale,output_dict=False): - device = features_a.device + def forward(self, image_features=None, text_features=None, text_a_features=None, text_b_features=None, logit_scale=None,output_dict=False): + + if features_a is not None and features_b is not None: + features_a = image_features + features_b = text_features + + elif text_a_features is not None and text_b_features is not None: + features_a = text_a_features + features_b = text_b_features + device = features_a.device if self.world_size > 1: all_features_a, all_features_b = gather_features( features_a, features_b, diff --git a/tests/test_TextTextCLIP.py b/tests/test_TextTextCLIP.py index dcd511d86..a68951927 100644 --- a/tests/test_TextTextCLIP.py +++ b/tests/test_TextTextCLIP.py @@ -11,10 +11,10 @@ os.environ["CUDA_VISIBLE_DEVICES"] = "" -@pytest.mark.parametrize("model_type", [("roberta-roberta")]) -def test_inference_simple(model_type, pretrained=None): - model, _, preprocess = open_clip.create_model_and_transforms(model_type, pretrained=pretrained, jit=False, text_to_text=True) - tokenizer = get_tokenizer(model_type) +@pytest.mark.parametrize("model_cfg", [("roberta-roberta")]) +def test_inference_simple(model_cfg, pretrained=None): + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='text_dual_encoder') + tokenizer = get_tokenizer(model_cfg) text_a = tokenizer(['this', 'is', 'a', 'document']) text_b = tokenizer(['this', 'is', 'a', 'summary']) diff --git a/tests/test_inference.py b/tests/test_inference.py index dca8dc44c..2e4b93c0b 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -29,7 +29,9 @@ 'mt5-xl-ViT-H-14', 'coca_base', 'coca_ViT-B-32', - 'coca_roberta-ViT-B-32' + 'coca_roberta-ViT-B-32', + 'roberta-roberta', + 'xlm-roberta-large-xlm-roberta-large', }) if 'OPEN_CLIP_TEST_REG_MODELS' in os.environ: From 26d677b11e8953b8778b64b6407696021e105ce3 Mon Sep 17 00:00:00 2001 From: lingjzhu <> Date: Mon, 30 Jan 2023 23:58:03 -0600 Subject: [PATCH 17/36] resolve conflicts --- src/training/data.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/training/data.py b/src/training/data.py index 371730093..b009c32fd 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -15,7 +15,6 @@ import torchvision.datasets as datasets import webdataset as wds from PIL import Image -from datasets import load_dataset from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler, IterableDataset, get_worker_info from torch.utils.data.distributed import DistributedSampler from webdataset.filters import _shuffle @@ -50,7 +49,9 @@ def __getitem__(self, idx): class HFTextPairDataset(Dataset): def __init__(self, input_filename, text_a_key, text_b_key, tokenizer=None): logging.debug(f'Loading parquet data from {input_filename}.') - + + from datasets import load_dataset + self.dataset = load_dataset(input_filename) self.text_a_key = text_a_key self.text_b_key = text_b_key From a3029f40c4f18eb3f2d071fa7b374ba92b0e6db1 Mon Sep 17 00:00:00 2001 From: lingjzhu <> Date: Tue, 31 Jan 2023 00:01:59 -0600 Subject: [PATCH 18/36] resolve conflicts --- src/open_clip/loss.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py index c9d9fbb51..197cc3b5a 100644 --- a/src/open_clip/loss.py +++ b/src/open_clip/loss.py @@ -87,9 +87,9 @@ def __init__( self.labels = {} - def forward(self, image_features=None, text_features=None, text_a_features=None, text_b_features=None, logit_scale=None,output_dict=False): + def forward(self, image_features=None, text_features=None, text_a_features=None, text_b_features=None, logit_scale=None, output_dict=False): - if features_a is not None and features_b is not None: + if image_features is not None and text_features is not None: features_a = image_features features_b = text_features From 633f53f83f47b0cd47a5b282eb546161fc1b3f37 Mon Sep 17 00:00:00 2001 From: lingjzhu <> Date: Tue, 31 Jan 2023 14:41:54 -0600 Subject: [PATCH 19/36] resolve conflicts in loss.py --- src/open_clip/factory.py | 2 +- src/open_clip/loss.py | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 5fe264e7d..d1b72b14a 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -52,7 +52,7 @@ def _rescan_model_configs(): _MODEL_CONFIGS[cf.stem] = model_cfg _MODEL_CONFIGS = {k: v for k, v in sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0]))} - print(_MODEL_CONFIGS) + _rescan_model_configs() # initial populate of model config registry diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py index 197cc3b5a..003d22bdb 100644 --- a/src/open_clip/loss.py +++ b/src/open_clip/loss.py @@ -87,15 +87,7 @@ def __init__( self.labels = {} - def forward(self, image_features=None, text_features=None, text_a_features=None, text_b_features=None, logit_scale=None, output_dict=False): - - if image_features is not None and text_features is not None: - features_a = image_features - features_b = text_features - - elif text_a_features is not None and text_b_features is not None: - features_a = text_a_features - features_b = text_b_features + def forward(self, features_a, features_b, logit_scale, output_dict=False): device = features_a.device if self.world_size > 1: @@ -159,7 +151,7 @@ def __init__( self.caption_loss = nn.CrossEntropyLoss(ignore_index=pad_id) def forward(self, image_features, text_features, logits, labels, logit_scale, output_dict=False): - clip_loss = super().forward(image_features, text_features, logit_scale) + clip_loss = super().forward(features_a=image_features, features_b=text_features, logit_scale=logit_scale) clip_loss = self.clip_loss_weight * clip_loss caption_loss = self.caption_loss( From 634709e4ab0f62a88f4432402e92835e1af867aa Mon Sep 17 00:00:00 2001 From: lingjzhu <> Date: Tue, 31 Jan 2023 14:45:27 -0600 Subject: [PATCH 20/36] resolve conflicts in loss.py --- src/open_clip/loss.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py index 003d22bdb..66f38ad69 100644 --- a/src/open_clip/loss.py +++ b/src/open_clip/loss.py @@ -87,7 +87,14 @@ def __init__( self.labels = {} - def forward(self, features_a, features_b, logit_scale, output_dict=False): + def forward(self, image_features=None, text_features=None, logit_scale=None, text_a_features=None, text_b_features=None, output_dict=False): + + if image_features is not None and text_features is not None: + features_a = image_features + features_b = text_features + elif text_a_features is not None and text_b_features is not None: + features_a = text_a_features + features_b = text_b_features device = features_a.device if self.world_size > 1: @@ -151,7 +158,7 @@ def __init__( self.caption_loss = nn.CrossEntropyLoss(ignore_index=pad_id) def forward(self, image_features, text_features, logits, labels, logit_scale, output_dict=False): - clip_loss = super().forward(features_a=image_features, features_b=text_features, logit_scale=logit_scale) + clip_loss = super().forward(image_features, text_features, logit_scale) clip_loss = self.clip_loss_weight * clip_loss caption_loss = self.caption_loss( From a9710b10ed87ae05a3a07b0f01e1de38620702d2 Mon Sep 17 00:00:00 2001 From: Jian Zhu Date: Mon, 6 Feb 2023 17:43:31 -0500 Subject: [PATCH 21/36] add output_dict --- src/open_clip/factory.py | 3 ++- src/open_clip/model.py | 16 ++++++++++++++++ .../model_configs/Siamese-xlm-roberta-large.json | 10 ++++++++++ src/training/data.py | 7 +++++-- src/training/main.py | 1 + src/training/params.py | 2 +- 6 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 src/open_clip/model_configs/Siamese-xlm-roberta-large.json diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index d1b72b14a..178e3c7d1 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -47,8 +47,9 @@ def _rescan_model_configs(): is_clip = all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg')) is_text_only = all(a in model_cfg for a in ('embed_dim', 'tower_a_cfg', 'tower_b_cfg')) + is_siamese_text_only = all(a in model_cfg for a in ('embed_dim', 'text_cfg')) is_multimodal = all(a in model_cfg for a in ('embed_dim', 'vision_cfg', 'text_cfg', 'multimodal_cfg')) - if is_clip or is_text_only or is_multimodal: + if is_clip or is_text_only or is_siamese_text_only or is_multimodal: _MODEL_CONFIGS[cf.stem] = model_cfg _MODEL_CONFIGS = {k: v for k, v in sorted(_MODEL_CONFIGS.items(), key=lambda x: _natural_key(x[0]))} diff --git a/src/open_clip/model.py b/src/open_clip/model.py index c0cc5e4a5..e1f5d8eb2 100644 --- a/src/open_clip/model.py +++ b/src/open_clip/model.py @@ -296,8 +296,10 @@ def __init__( tower_b_cfg: CLIPTextCfg, quick_gelu: bool = False, cast_dtype: Optional[torch.dtype] = None, + output_dict: bool = False, ): super().__init__() + self.output_dict = output_dict self.tower_a = _build_text_tower(embed_dim, tower_a_cfg, quick_gelu, cast_dtype) self.tower_b = _build_text_tower(embed_dim, tower_b_cfg, quick_gelu, cast_dtype) self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) @@ -324,6 +326,12 @@ def encode_text_b(self, text, normalize: bool = False): def forward(self, text_a, text_b): features_a = self.encode_text_a(text_a, normalize=True) features_b = self.encode_text_b(text_b, normalize=True) + if self.output_dict: + return { + "text_a_features": features_a, + "text_b_features": features_b, + "logit_scale": self.logit_scale.exp() + } return features_a, features_b, self.logit_scale.exp() @@ -336,8 +344,10 @@ def __init__( text_cfg: CLIPTextCfg, quick_gelu: bool = False, cast_dtype: Optional[torch.dtype] = None, + output_dict: bool = False, ): super().__init__() + self.output_dict = output_dict self.text_tower = _build_text_tower(embed_dim, text_cfg, quick_gelu, cast_dtype) self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) @@ -355,6 +365,12 @@ def encode_text(self, text, normalize: bool = False): def forward(self, text_a, text_b): features_a = self.encode_text(text_a, normalize=True) features_b = self.encode_text(text_b, normalize=True) + if self.output_dict: + return { + "text_a_features": features_a, + "text_b_features": features_b, + "logit_scale": self.logit_scale.exp() + } return features_a, features_b, self.logit_scale.exp() diff --git a/src/open_clip/model_configs/Siamese-xlm-roberta-large.json b/src/open_clip/model_configs/Siamese-xlm-roberta-large.json new file mode 100644 index 000000000..037a97d8a --- /dev/null +++ b/src/open_clip/model_configs/Siamese-xlm-roberta-large.json @@ -0,0 +1,10 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "text_cfg": { + "hf_model_name": "xlm-roberta-large", + "hf_tokenizer_name": "xlm-roberta-large", + "proj": "mlp", + "pooler_type": "mean_pooler" + } +} \ No newline at end of file diff --git a/src/training/data.py b/src/training/data.py index b009c32fd..83f1624bf 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -48,11 +48,13 @@ def __getitem__(self, idx): class HFTextPairDataset(Dataset): def __init__(self, input_filename, text_a_key, text_b_key, tokenizer=None): - logging.debug(f'Loading parquet data from {input_filename}.') + logging.debug(f'Loading data from {input_filename}.') from datasets import load_dataset self.dataset = load_dataset(input_filename) + if "train" in self.dataset.keys(): + self.dataset = self.dataset['train'] self.text_a_key = text_a_key self.text_b_key = text_b_key @@ -61,7 +63,7 @@ def __init__(self, input_filename, text_a_key, text_b_key, tokenizer=None): self.tokenize = tokenizer def __len__(self): - return len(self.text_a) + return len(self.dataset[self.text_a_key]) def __getitem__(self, idx): texts_a = self.tokenize([str(self.dataset[self.text_a_key][idx])])[0] @@ -88,6 +90,7 @@ def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None ) num_samples = len(dataset) + logging.debug("%s"%(num_samples)) sampler = DistributedSampler(dataset) if args.distributed and is_train else None shuffle = is_train and sampler is None diff --git a/src/training/main.py b/src/training/main.py index 269adee92..803d48c2f 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -324,6 +324,7 @@ def main(args): # initialize datasets data = get_data(args, (preprocess_train, preprocess_val), epoch=start_epoch, tokenizer=get_tokenizer(args.model)) assert len(data), 'At least one train or eval dataset must be specified.' + print(len(data)) # create scheduler if train scheduler = None diff --git a/src/training/params.py b/src/training/params.py index 683017396..f148c4030 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -413,7 +413,7 @@ def parse_args(args): parser.add_argument( "--log-every-n-steps", type=int, - default=100, + default=10, help="Log every n steps to tensorboard/console/wandb.", ) parser.add_argument( From 4084147b6de66791549d91106ec2055e76016917 Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Sun, 19 Feb 2023 18:29:13 -0500 Subject: [PATCH 22/36] add webdataset loader --- src/training/data.py | 116 +++++++++++++++++-------------------------- 1 file changed, 45 insertions(+), 71 deletions(-) diff --git a/src/training/data.py b/src/training/data.py index 83f1624bf..59b993e93 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd +from functools import partial import torch import torchvision.datasets as datasets import webdataset as wds @@ -26,73 +27,24 @@ hvd = None -class TextPairDataset(Dataset): - def __init__(self, input_filename, text_a_key, text_b_key, tokenizer=None): - logging.debug(f'Loading parquet data from {input_filename}.') - df = pd.read_parquet(input_filename) - - self.text_a = df[text_a_key].tolist() - self.text_b = df[text_b_key].tolist() - logging.debug('Done loading data.') - - self.tokenize = tokenizer - - def __len__(self): - return len(self.text_a) - - def __getitem__(self, idx): - texts_a = self.tokenize([str(self.text_a[idx])])[0] - texts_b = self.tokenize([str(self.text_b[idx])])[0] - return texts_a,texts_b - - -class HFTextPairDataset(Dataset): - def __init__(self, input_filename, text_a_key, text_b_key, tokenizer=None): - logging.debug(f'Loading data from {input_filename}.') - - from datasets import load_dataset - - self.dataset = load_dataset(input_filename) - if "train" in self.dataset.keys(): - self.dataset = self.dataset['train'] - self.text_a_key = text_a_key - self.text_b_key = text_b_key - - logging.debug('Done loading data.') - - self.tokenize = tokenizer - - def __len__(self): - return len(self.dataset[self.text_a_key]) - - def __getitem__(self, idx): - texts_a = self.tokenize([str(self.dataset[self.text_a_key][idx])])[0] - texts_b = self.tokenize([str(self.dataset[self.text_b_key][idx])])[0] - return texts_a,texts_b -def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None): +def get_HF_text_dataset(args, preprocess_fn, is_train, epoch=0, floor=False, tokenizer=None): input_filename = args.train_data if is_train else args.val_data assert input_filename - if input_filename.endswith('.parquet'): - dataset = TextPairDataset( - input_filename, - text_a_key=args.text_a_key, - text_b_key=args.text_b_key, - tokenizer=tokenizer - ) - else: - dataset = HFTextPairDataset( - input_filename, - text_a_key=args.text_a_key, - text_b_key=args.text_b_key, - tokenizer=tokenizer - ) - - num_samples = len(dataset) - logging.debug("%s"%(num_samples)) + + from datasets import load_dataset + dataset = load_dataset(input_filename,streaming=True) + if "train" in dataset.keys(): + dataset = dataset['train'] + sampler = DistributedSampler(dataset) if args.distributed and is_train else None shuffle = is_train and sampler is None + + def fetch(text_a_key,text_b_key,tokenizer,batch): + texts_a = tokenizer.tokenize([text for text in batch[text_a_key]]) + texts_b = tokenizer.tokenize([text for text in batch[text_b_key]]) + return texts_a,texts_b dataloader = DataLoader( dataset, @@ -102,9 +54,20 @@ def get_text_pair_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None pin_memory=True, sampler=sampler, drop_last=is_train, + collate_fn=partial(fetch,args.text_a_key,args.text_b_key,tokenizer), ) + + num_samples = args.train_num_samples + round_fn = math.floor if floor else math.ceil + global_batch_size = args.batch_size * args.world_size + num_batches = round_fn(num_samples / global_batch_size) + num_workers = max(1, args.workers) + num_worker_batches = round_fn(num_batches / num_workers) # per dataloader worker + num_batches = num_worker_batches * num_workers + + logging.debug("%s"%(num_samples)) dataloader.num_samples = num_samples - dataloader.num_batches = len(dataloader) + dataloader.num_batches = num_batches return DataInfo(dataloader, sampler) @@ -430,14 +393,24 @@ def get_wds_dataset(args, preprocess_img, is_train, epoch=0, floor=False, tokeni # at this point, we have an iterator over the shards assigned to each worker wds.tarfile_to_samples(handler=log_and_continue), ]) - pipeline.extend([ - wds.select(filter_no_caption_or_no_image), - wds.decode("pilrgb", handler=log_and_continue), - wds.rename(image="jpg;png;jpeg;webp", text="txt"), - wds.map_dict(image=preprocess_img, text=lambda text: tokenizer(text)[0]), - wds.to_tuple("image", "text"), - wds.batched(args.batch_size, partial=not is_train), - ]) + + if 'text' in args.model_type: + pipeline.extend([ + wds.to_tuple("text_a", "text_b"), + wds.map_tuple(lambda text: tokenizer(text.decode('utf8'))[0], lambda text: tokenizer(text.decode('utf8'))[0]), + wds.batched(args.batch_size, partial=not is_train), + ]) + + + else: + pipeline.extend([ + wds.select(filter_no_caption_or_no_image), + wds.decode("pilrgb", handler=log_and_continue), + wds.rename(image="jpg;png;jpeg;webp", text="txt"), + wds.map_dict(image=preprocess_img, text=lambda text: tokenizer(text)[0]), + wds.to_tuple("image", "text"), + wds.batched(args.batch_size, partial=not is_train), + ]) dataset = wds.DataPipeline(*pipeline) if is_train: @@ -456,6 +429,7 @@ def get_wds_dataset(args, preprocess_img, is_train, epoch=0, floor=False, tokeni # last batches are partial, eval is done on single (master) node num_batches = math.ceil(num_samples / args.batch_size) + dataloader = wds.WebLoader( dataset, batch_size=None, @@ -560,7 +534,7 @@ def get_synthetic_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None def get_dataset_fn(data_path, dataset_type): if dataset_type == 'textpair': - return get_text_pair_dataset + return get_HF_text_dataset if dataset_type == "webdataset": return get_wds_dataset elif dataset_type == "csv": From 91679762b177a917757096d5c77ba96a61537a36 Mon Sep 17 00:00:00 2001 From: jzhu <7823218+lingjzhu@users.noreply.github.com> Date: Mon, 20 Mar 2023 17:30:51 -0700 Subject: [PATCH 23/36] Update loss.py --- src/open_clip/loss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py index 3fd5088d0..8737d2346 100644 --- a/src/open_clip/loss.py +++ b/src/open_clip/loss.py @@ -132,7 +132,7 @@ def forward(self, image_features=None, text_features=None, logit_scale=None, tex device = features_a.device logits_per_feature_a, logits_per_feature_b = self.get_logits(features_a, features_b, logit_scale) - labels = self.get_ground_truth(device, logits_per_image.shape[0]) + labels = self.get_ground_truth(device, logits_per_feature_a.shape[0]) total_loss = ( F.cross_entropy(logits_per_feature_a, labels) + From 579a5913a2938e353af3d2dbefb0bb9ba42a0a27 Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Thu, 23 Mar 2023 00:54:41 -0400 Subject: [PATCH 24/36] add sts evaluation code --- docs/script_examples/text_example.sh | 59 ++++++++++++++++++++++++++++ requirements-training.txt | 1 + src/training/data.py | 45 ++++++++++++++------- src/training/main.py | 3 +- src/training/params.py | 8 +++- src/training/sts_eval.py | 56 ++++++++++++++++++++++++++ src/training/train.py | 11 ++++-- 7 files changed, 164 insertions(+), 19 deletions(-) create mode 100644 docs/script_examples/text_example.sh create mode 100644 src/training/sts_eval.py diff --git a/docs/script_examples/text_example.sh b/docs/script_examples/text_example.sh new file mode 100644 index 000000000..faba97f30 --- /dev/null +++ b/docs/script_examples/text_example.sh @@ -0,0 +1,59 @@ +#!/bin/bash +#SBATCH --partition=g80 +#SBATCH --account=laion +#SBATCH --job-name=TextTextCLIP +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=4 +#SBATCH --cpus-per-task=4 +#SBATCH --output=logs/%x_%j.out +#SBATCH --comment=laion +#SBATCH --open-mode=append +#SBATCH --exclusive + +module load openmpi +module load cuda/11.7 + +export PYTHONFAULTHANDLER=1 +export CUDA_LAUNCH_BLOCKING=0 +export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"` +export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1) +export MASTER_PORT=12802 +export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l` + +echo go $COUNT_NODE +echo $HOSTNAMES + + +cd /admin/home-jianz/open_clip/src +export PYTHONPATH="$PYTHONPATH:/admin/home-jianz/open_clip/src" + +EXP_NAME="" + +#srun --comment laion --cpu_bind=v --accel-bind=gn torchrun --nproc_per_node 4 --max_restarts=3 --rdzv_id=1 --rdzv_backend=c10d --rdzv_endpoint=localhost:0 -m training.main \ +srun --comment laion --cpu_bind=v --accel-bind=gn python3 -m training.main \ + --save-frequency 1 \ + --dataset-type="webdataset" \ + --text-a-key="text_a" \ + --text-b-key="text_b" \ + --report-to wandb \ + --wandb-project-name="TextTextCLIP" \ + --train-data="/fsx/home-jianz/mycache/huggingface/hub/datasets--lingjzhu--laion-multi-2B/snapshots/7ec0d572ac4d8da1e6997ed32e383ab63967e05d/laion-multi-2B-{000..127}.tar" \ + --train-num-samples 135646078 \ + --warmup 2000 \ + --batch-size=2048 \ + --precision amp_bfloat16 \ + --lr=0.001 \ + --wd=0.2 \ + --epochs=97 \ + --workers=1 \ + --model="Siamese-xlm-roberta-large" \ + --seed 0 \ + --log-every-n-steps 5 \ + --local-loss \ + --gather-with-grad \ + --ddp-static-graph \ + --grad-checkpointing \ + --model_type="text_siamese_encoder" \ + --debug \ + --sts-val-data="lingjzhu/sts17-crosslingual" + diff --git a/requirements-training.txt b/requirements-training.txt index c44eb61d7..4e8ff3d1f 100644 --- a/requirements-training.txt +++ b/requirements-training.txt @@ -8,5 +8,6 @@ pandas braceexpand huggingface_hub transformers +datasets timm fsspec diff --git a/src/training/data.py b/src/training/data.py index f8f0823dd..256472896 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -28,25 +28,40 @@ hvd = None +class STSDataset(Dataset): + def __init__(self, input_filename, tokenizer=None): + from datasets import load_dataset + logging.debug(f'Loading STS data from {input_filename}.') + self.dataset = load_dataset(input_filename,keep_in_memory=True) + if "train" in self.dataset.keys(): + self.dataset = self.dataset['train'] + logging.debug('Done loading data.') -def get_HF_text_dataset(args, preprocess_fn, is_train, epoch=0, floor=False, tokenizer=None): - input_filename = args.train_data if is_train else args.val_data + self.tokenize = tokenizer + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + entry = self.dataset[idx] + sent1 = self.tokenize([str(entry['sentence1'])])[0] + sent2 = self.tokenize([str(entry['sentence2'])])[0] + score = entry['score'] + return sent1,sent2,score + + + +def get_STS_dataset(args, preprocess_fn, is_train, epoch=0, floor=False, tokenizer=None): + input_filename = args.sts_val_data assert input_filename - from datasets import load_dataset - dataset = load_dataset(input_filename,streaming=True) - if "train" in dataset.keys(): - dataset = dataset['train'] + + dataset = STSDataset(input_filename,tokenizer=tokenizer) sampler = DistributedSampler(dataset) if args.distributed and is_train else None shuffle = is_train and sampler is None - def fetch(text_a_key,text_b_key,tokenizer,batch): - texts_a = tokenizer.tokenize([text for text in batch[text_a_key]]) - texts_b = tokenizer.tokenize([text for text in batch[text_b_key]]) - return texts_a,texts_b - dataloader = DataLoader( dataset, batch_size=args.batch_size, @@ -55,7 +70,6 @@ def fetch(text_a_key,text_b_key,tokenizer,batch): pin_memory=True, sampler=sampler, drop_last=is_train, - collate_fn=partial(fetch,args.text_a_key,args.text_b_key,tokenizer), ) num_samples = args.train_num_samples @@ -565,8 +579,8 @@ def get_synthetic_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=None def get_dataset_fn(data_path, dataset_type): - if dataset_type == 'textpair': - return get_HF_text_dataset + if dataset_type == 'sts': + return get_STS_dataset if dataset_type == "webdataset": return get_wds_dataset elif dataset_type == "csv": @@ -603,5 +617,8 @@ def get_data(args, preprocess_fns, epoch=0, tokenizer=None): if args.imagenet_v2 is not None: data["imagenet-v2"] = get_imagenet(args, preprocess_fns, "v2") + + if args.sts_val_data is not None: + data['sts-val'] = get_dataset_fn(args,'sts')(args, preprocess_fns, is_train=False, epoch=epoch, tokenizer=tokenizer) return data diff --git a/src/training/main.py b/src/training/main.py index 298e06206..d23917cfd 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -408,7 +408,8 @@ def main(args): train_one_epoch(model, data, loss, epoch, optimizer, scaler, scheduler, dist_model, args, tb_writer=writer) completed_epoch = epoch + 1 - if any(v in data for v in ('val', 'imagenet-val', 'imagenet-v2')): + if any(v in data for v in ('val', 'imagenet-val', 'imagenet-v2', 'sts-val')): + logging.info('Running evaluation.') evaluate(model, data, completed_epoch, args, writer) # Saving checkpoints. diff --git a/src/training/params.py b/src/training/params.py index 4759a5997..a0640f456 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -113,6 +113,12 @@ def parse_args(args): default=None, help="Path to imagenet v2 for conducting zero shot evaluation.", ) + parser.add_argument( + "--sts-val-data", + type=str, + default=None, + help="Path to STS17 crosslingual evaluation data. The default path is: lingjzhu/sts17-crosslingual", + ) parser.add_argument( "--logs", type=str, @@ -427,7 +433,7 @@ def parse_args(args): help="Log every n steps to tensorboard/console/wandb.", ) parser.add_argument( - "--model_type", + "--model-type", type=str, choices=["CLIP", "text_dual_encoder", "text_siamese_encoder"], default="CLIP", diff --git a/src/training/sts_eval.py b/src/training/sts_eval.py new file mode 100644 index 000000000..ec677b370 --- /dev/null +++ b/src/training/sts_eval.py @@ -0,0 +1,56 @@ +import torch +import torch.nn.functional as F +import open_clip +from datasets import load_dataset +from scipy.stats import spearmanr, pearsonr +from tqdm import tqdm +import numpy as np + +import logging + +from open_clip import get_cast_dtype, get_tokenizer +from .precision import get_autocast + +def run(model,dataloader,args): + + preds = [] + gt_scores = [] + + for sent1,sent2,scores in tqdm(dataloader, unit_scale=args.batch_size): + + with torch.no_grad(), torch.cuda.amp.autocast(): + a_features = model.encode_text(sent1.to(args.device)) + b_features = model.encode_text(sent2.to(args.device)) + + pred = F.cosine_similarity(a_features,b_features,dim=-1).squeeze() + + preds.append(pred.cpu().detach().numpy()) + gt_scores.append(scores) + + preds = np.concatenate(preds,axis=0) + gt_scores = np.concatenate(gt_scores,axis=0) + + return spearmanr(preds,gt_scores)[0], pearsonr(preds,gt_scores)[0] + + +def sts_eval(model, data, epoch, args): + + if 'sts-val' not in data: + return {} + + logging.info('Starting STS-17 evaluation.') + + + results = {} + + spearman_corr, pearson_corr = run(model,data['sts-val'].dataloader, args) + results['sts-val-spearman-corr'] = spearman_corr + results['sts-val-pearson-corr'] = pearson_corr + + logging.info('Finished STS-17') + + return results + + + + diff --git a/src/training/train.py b/src/training/train.py index e0a140f9c..9edba64db 100644 --- a/src/training/train.py +++ b/src/training/train.py @@ -17,6 +17,7 @@ from open_clip import get_cast_dtype, CLIP, CustomTextCLIP from .distributed import is_master from .zero_shot import zero_shot_eval +from .sts_eval import sts_eval from .precision import get_autocast @@ -239,9 +240,13 @@ def evaluate(model, data, epoch, args, tb_writer=None): return metrics device = torch.device(args.device) model.eval() - - zero_shot_metrics = zero_shot_eval(model, data, epoch, args) - metrics.update(zero_shot_metrics) + + if hasattr(model,'visual'): + zero_shot_metrics = zero_shot_eval(model, data, epoch, args) + metrics.update(zero_shot_metrics) + elif hasattr(model,'text_tower'): + sts_metrics = sts_eval(model, data, epoch, args) + metrics.update(sts_metrics) autocast = get_autocast(args.precision) cast_dtype = get_cast_dtype(args.precision) From 646d517294ad3935ea33ca3693c7515468d112dd Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Thu, 23 Mar 2023 01:07:58 -0400 Subject: [PATCH 25/36] fix dependencies --- requirements-training.txt | 1 + tests/test_TextTextCLIP.py | 21 --------------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/requirements-training.txt b/requirements-training.txt index 4e8ff3d1f..70f76fc29 100644 --- a/requirements-training.txt +++ b/requirements-training.txt @@ -11,3 +11,4 @@ transformers datasets timm fsspec +scipy diff --git a/tests/test_TextTextCLIP.py b/tests/test_TextTextCLIP.py index a68951927..686f022ff 100644 --- a/tests/test_TextTextCLIP.py +++ b/tests/test_TextTextCLIP.py @@ -29,24 +29,3 @@ def test_inference_simple(model_cfg, pretrained=None): text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) print(text_probs) -''' -if __name__ == "__main__": - - - main([ - '--save-frequency', '1', - '--train-num-samples', '16', - '--warmup', '1', - '--batch-size', '2', - '--lr', '1e-3', - '--wd', '0.1', - '--epochs', '1', - '--workers', '2', - '--model', 'roberta-roberta', - '--dataset-type', 'textpair', - '--train-data', 'text_pairs.parquet.gzip', - '--text-to-text', 'True', - '--text-a-key', 'query', - '--text-b-key', 'doc' - ]) -''' \ No newline at end of file From 9e5ed73339feed7e97ae3775cb9a493a4f26f272 Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Sun, 2 Apr 2023 14:46:10 -0400 Subject: [PATCH 26/36] add weighted mean pooling for decoder models --- src/open_clip/factory.py | 3 + src/open_clip/hf_configs.py | 14 +++++ src/open_clip/hf_model.py | 42 +++++++++++--- .../model_configs/Siamese-pythia-410m.json | 10 ++++ .../model_configs/pythia-1b-pythia-1b.json | 16 ++++++ .../pythia-410m-pythia-410m.json | 16 ++++++ .../xlm-roberta-large-xlm-roberta-large.json | 16 ------ tests/test_TextTextCLIP.py | 56 +++++++++++++++++++ 8 files changed, 150 insertions(+), 23 deletions(-) create mode 100644 src/open_clip/model_configs/Siamese-pythia-410m.json create mode 100644 src/open_clip/model_configs/pythia-1b-pythia-1b.json create mode 100644 src/open_clip/model_configs/pythia-410m-pythia-410m.json delete mode 100644 src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 36bcaf20c..6885858ba 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -91,6 +91,9 @@ def get_tokenizer(model_name): elif 'tower_a_cfg' in config.keys(): key = 'tower_a_cfg' tokenizer = HFTokenizer(config[key]['hf_tokenizer_name']) if 'hf_tokenizer_name' in config[key] else tokenize + + if "pythia" in config[key]['hf_tokenizer_name']: + tokenizer.tokenizer.pad_token_id = 1 return tokenizer diff --git a/src/open_clip/hf_configs.py b/src/open_clip/hf_configs.py index e236222ba..41fd5a68b 100644 --- a/src/open_clip/hf_configs.py +++ b/src/open_clip/hf_configs.py @@ -42,4 +42,18 @@ }, "pooler": "mean_pooler", }, + # https://huggingface.co/docs/transformers/main/en/model_doc/gpt_neox#transformers.GPTNeoXConfig + "gpt_neox": { + "config_names": { + # https://github.com/huggingface/transformers/blob/c612628045822f909020f7eb6784c79700813eda/src/transformers/models/gpt_neox/modeling_gpt_neox.py#L410 + "context_length": "max_position_embeddings", + "vocab_size": "vocab_size", + "width": "hidden_size", + "heads": "num_attention_heads", + "layers": "num_hidden_layers", + "layer_attr": "layers", + "token_embeddings_attr": "embed_in" + }, + "pooler": "weighted_mean_pooler", + }, } diff --git a/src/open_clip/hf_model.py b/src/open_clip/hf_model.py index fbccc8127..e38ef30c5 100644 --- a/src/open_clip/hf_model.py +++ b/src/open_clip/hf_model.py @@ -61,6 +61,29 @@ def forward(self, x: BaseModelOutput, attention_mask: TensorType): return masked_output.max(1).values +@register_pooler +class WeightedMeanPooler(nn.Module): + """Weighted mean pooling for autoregressive models""" + + def forward(self, x: BaseModelOutput, attention_mask: TensorType): + + input_mask_expanded = attention_mask.unsqueeze(-1).expand(x.last_hidden_state.size()).float() + weights = ( + torch.arange(start=1, end=x.last_hidden_state.shape[1] + 1) + .unsqueeze(0) + .unsqueeze(-1) + .expand(x.last_hidden_state.size()) + .float().to(x.last_hidden_state.device) + ) + assert weights.shape == x.last_hidden_state.shape == input_mask_expanded.shape + input_mask_expanded = input_mask_expanded * weights + + sum_embeddings = torch.sum(x.last_hidden_state * input_mask_expanded, 1) + sum_mask = input_mask_expanded.sum(1) + sum_mask = torch.clamp(sum_mask, min=1e-9) + return sum_embeddings / sum_mask + + @register_pooler class ClsPooler(nn.Module): """CLS token pooling""" @@ -78,6 +101,8 @@ def forward(self, x: BaseModelOutput, attention_mask: TensorType): return x.pooler_output return x.last_hidden_state[:, self.cls_token_position, :] + + class HFTextEncoder(nn.Module): @@ -111,6 +136,9 @@ def __init__( if hasattr(self.config, "is_encoder_decoder") and self.config.is_encoder_decoder: self.transformer = create_func(model_args) self.transformer = self.transformer.encoder + elif 'gpt' in self.config.model_type: + self.transformer = create_func(model_args) + self.config.pad_token_id = 1 # this is for GPT-NeoX. It might need to be changed if other models are needed. else: self.transformer = create_func(model_args, add_pooling_layer=uses_transformer_pooler) else: @@ -139,16 +167,16 @@ def forward(self, x: TensorType): out = self.transformer(input_ids=x, attention_mask=attn_mask) pooled_out = self.pooler(out, attn_mask) projected = self.proj(pooled_out) - - seq_len = out.last_hidden_state.shape[1] - tokens = ( - out.last_hidden_state[:, torch.arange(seq_len) != self.pooler.cls_token_position, :] - if type(self.pooler) == ClsPooler - else out.last_hidden_state - ) if self.output_tokens: + seq_len = out.last_hidden_state.shape[1] + tokens = ( + out.last_hidden_state[:, torch.arange(seq_len) != self.pooler.cls_token_position, :] + if type(self.pooler) == ClsPooler + else out.last_hidden_state + ) return projected, tokens + return projected def lock(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): diff --git a/src/open_clip/model_configs/Siamese-pythia-410m.json b/src/open_clip/model_configs/Siamese-pythia-410m.json new file mode 100644 index 000000000..1c86bfabb --- /dev/null +++ b/src/open_clip/model_configs/Siamese-pythia-410m.json @@ -0,0 +1,10 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "text_cfg": { + "hf_model_name": "EleutherAI/pythia-410m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-410m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/model_configs/pythia-1b-pythia-1b.json b/src/open_clip/model_configs/pythia-1b-pythia-1b.json new file mode 100644 index 000000000..bbd477636 --- /dev/null +++ b/src/open_clip/model_configs/pythia-1b-pythia-1b.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "tower_a_cfg": { + "hf_model_name": "EleutherAI/pythia-1b-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-1b-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + }, + "tower_b_cfg": { + "hf_model_name": "EleutherAI/pythia-1b-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-1b-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/model_configs/pythia-410m-pythia-410m.json b/src/open_clip/model_configs/pythia-410m-pythia-410m.json new file mode 100644 index 000000000..e5931980f --- /dev/null +++ b/src/open_clip/model_configs/pythia-410m-pythia-410m.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "tower_a_cfg": { + "hf_model_name": "EleutherAI/pythia-410m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-410m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + }, + "tower_b_cfg": { + "hf_model_name": "EleutherAI/pythia-410m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-410m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json b/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json deleted file mode 100644 index 766e71439..000000000 --- a/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "embed_dim": 512, - "quick_gelu": true, - "tower_a_cfg": { - "hf_model_name": "xlm-roberta-large", - "hf_tokenizer_name": "xlm-roberta-large", - "proj": "mlp", - "pooler_type": "mean_pooler" - }, - "tower_b_cfg": { - "hf_model_name": "xlm-roberta-large", - "hf_tokenizer_name": "xlm-roberta-large", - "proj": "mlp", - "pooler_type": "mean_pooler" - } -} \ No newline at end of file diff --git a/tests/test_TextTextCLIP.py b/tests/test_TextTextCLIP.py index 686f022ff..d9df60422 100644 --- a/tests/test_TextTextCLIP.py +++ b/tests/test_TextTextCLIP.py @@ -29,3 +29,59 @@ def test_inference_simple(model_cfg, pretrained=None): text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) print(text_probs) + +@pytest.mark.parametrize("model_cfg", [("Siamese-xlm-roberta-large")]) +def test_inference_simple(model_cfg, pretrained=None): + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='text_siamese_encoder') + tokenizer = get_tokenizer(model_cfg) + + text_a = tokenizer(['this', 'is', 'a', 'document']) + text_b = tokenizer(['this', 'is', 'a', 'summary']) + + with torch.no_grad(): + text_a_features = model.encode_text(text_a) + text_b_features = model.encode_text(text_b) + + print(text_a_features.shape) + print(text_b_features.shape) + + text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) + print(text_probs) + + +@pytest.mark.parametrize("model_cfg", [("pythia-1b-pythia-1b")]) +def test_inference_simple(model_cfg, pretrained=None): + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='text_dual_encoder') + tokenizer = get_tokenizer(model_cfg) + + text_a = tokenizer(['this', 'is', 'a', 'document']) + text_b = tokenizer(['this', 'is', 'a', 'summary']) + + with torch.no_grad(): + text_a_features = model.encode_text_a(text_a) + text_b_features = model.encode_text_b(text_b) + + print(text_a_features.shape) + print(text_b_features.shape) + + text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) + print(text_probs) + + +@pytest.mark.parametrize("model_cfg", [("Siamese-pythia-410m")]) +def test_inference_simple(model_cfg, pretrained=None): + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='text_siamese_encoder') + tokenizer = get_tokenizer(model_cfg) + + text_a = tokenizer(['this', 'is', 'a', 'document']) + text_b = tokenizer(['this', 'is', 'a', 'summary']) + + with torch.no_grad(): + text_a_features = model.encode_text(text_a) + text_b_features = model.encode_text(text_b) + + print(text_a_features.shape) + print(text_b_features.shape) + + text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) + print(text_probs) From 2ede8fc41287c7e55c376b70e91038d2d94312dd Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Sun, 2 Apr 2023 15:35:28 -0400 Subject: [PATCH 27/36] fix tokenizers --- src/open_clip/factory.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 6885858ba..630adcfb4 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -90,10 +90,13 @@ def get_tokenizer(model_name): key = 'text_cfg' elif 'tower_a_cfg' in config.keys(): key = 'tower_a_cfg' - tokenizer = HFTokenizer(config[key]['hf_tokenizer_name']) if 'hf_tokenizer_name' in config[key] else tokenize + if 'hf_tokenizer_name' in config[key]: + tokenizer = HFTokenizer(config[key]['hf_tokenizer_name']) + if "pythia" in config[key]['hf_tokenizer_name']: + tokenizer.tokenizer.pad_token_id = 1 + else: + tokenizer = tokenize - if "pythia" in config[key]['hf_tokenizer_name']: - tokenizer.tokenizer.pad_token_id = 1 return tokenizer From 5415590addf58002449d0a6695f8dcb97873ca44 Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Sun, 2 Apr 2023 19:35:28 -0400 Subject: [PATCH 28/36] enable freezing all weights but biases --- src/open_clip/hf_model.py | 9 +++++-- src/open_clip/model.py | 16 ++++++------ .../model_configs/pythia-1b-pythia-1b.json | 16 ------------ src/training/main.py | 9 ++++--- src/training/params.py | 26 ++++++++++++++++--- tests/test_TextTextCLIP.py | 8 +++++- 6 files changed, 50 insertions(+), 34 deletions(-) delete mode 100644 src/open_clip/model_configs/pythia-1b-pythia-1b.json diff --git a/src/open_clip/hf_model.py b/src/open_clip/hf_model.py index e38ef30c5..29af8b9ea 100644 --- a/src/open_clip/hf_model.py +++ b/src/open_clip/hf_model.py @@ -179,11 +179,16 @@ def forward(self, x: TensorType): return projected - def lock(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): + def lock(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): if not unlocked_layers: # full freezing for n, p in self.transformer.named_parameters(): - p.requires_grad = (not freeze_layer_norm) if "LayerNorm" in n.split(".") else False + p.requires_grad = False + if "LayerNorm" in n.split("."): + p.requires_grad = (not freeze_layer_norm) + if 'bias' in n.split("."): + p.requires_grad = unlocked_biases return + encoder = self.transformer.encoder if hasattr(self.transformer, 'encoder') else self.transformer layer_list = getattr(encoder, arch_dict[self.config.model_type]["config_names"]["layer_attr"]) diff --git a/src/open_clip/model.py b/src/open_clip/model.py index 0ee88fafe..998dd6c11 100644 --- a/src/open_clip/model.py +++ b/src/open_clip/model.py @@ -261,8 +261,8 @@ def lock_image_tower(self, unlocked_groups=0, freeze_bn_stats=False): # lock image tower as per LiT - https://arxiv.org/abs/2111.07991 self.visual.lock(unlocked_groups=unlocked_groups, freeze_bn_stats=freeze_bn_stats) - def lock_text_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): - self.text.lock(unlocked_layers, freeze_layer_norm) + def lock_text_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): + self.text.lock(unlocked_layers, freeze_layer_norm, unlocked_biases) @torch.jit.ignore def set_grad_checkpointing(self, enable=True): @@ -306,11 +306,11 @@ def __init__( self.tower_b = _build_text_tower(embed_dim, tower_b_cfg, quick_gelu, cast_dtype) self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) - def lock_tower_a(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): - self.tower_a.lock(unlocked_layers, freeze_layer_norm) + def lock_tower_a(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): + self.tower_a.lock(unlocked_layers, freeze_layer_norm, unlocked_biases) - def lock_tower_b(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): - self.tower_b.lock(unlocked_layers, freeze_layer_norm) + def lock_tower_b(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): + self.tower_b.lock(unlocked_layers, freeze_layer_norm, unlocked_biases) @torch.jit.ignore def set_grad_checkpointing(self, enable=True): @@ -353,8 +353,8 @@ def __init__( self.text_tower = _build_text_tower(embed_dim, text_cfg, quick_gelu, cast_dtype) self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) - def lock_text_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): - self.text_tower.lock(unlocked_layers, freeze_layer_norm) + def lock_text_tower(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True, unlocked_biases: bool = False): + self.text_tower.lock(unlocked_layers, freeze_layer_norm, unlocked_biases) @torch.jit.ignore def set_grad_checkpointing(self, enable=True): diff --git a/src/open_clip/model_configs/pythia-1b-pythia-1b.json b/src/open_clip/model_configs/pythia-1b-pythia-1b.json deleted file mode 100644 index bbd477636..000000000 --- a/src/open_clip/model_configs/pythia-1b-pythia-1b.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "embed_dim": 512, - "quick_gelu": true, - "tower_a_cfg": { - "hf_model_name": "EleutherAI/pythia-1b-deduped", - "hf_tokenizer_name": "EleutherAI/pythia-1b-deduped", - "proj": "mlp", - "pooler_type": "weighted_mean_pooler" - }, - "tower_b_cfg": { - "hf_model_name": "EleutherAI/pythia-1b-deduped", - "hf_tokenizer_name": "EleutherAI/pythia-1b-deduped", - "proj": "mlp", - "pooler_type": "weighted_mean_pooler" - } -} \ No newline at end of file diff --git a/src/training/main.py b/src/training/main.py index d23917cfd..19dbea851 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -255,15 +255,18 @@ def main(args): if args.lock_text: model.lock_text_tower( unlocked_layers=args.lock_text_unlocked_layers, - freeze_layer_norm=args.lock_text_freeze_layer_norm) + freeze_layer_norm=args.lock_text_freeze_layer_norm, + unlocked_biases=args.text_unlocked_biase) if args.lock_tower_a: model.lock_tower_a( unlocked_layers=args.lock_tower_a_unlocked_layers, - freeze_layer_norm=args.lock_tower_a_freeze_layer_norm) + freeze_layer_norm=args.lock_tower_a_freeze_layer_norm, + unlocked_biases=args.text_a_unlocked_biases) if args.lock_tower_b: model.lock_tower_b( unlocked_layers=args.lock_tower_b_unlocked_layers, - freeze_layer_norm=args.lock_tower_b_freeze_layer_norm) + freeze_layer_norm=args.lock_tower_b_freeze_layer_norm, + unlocked_biases=args.text_b_unlocked_biases) if args.grad_checkpointing: model.set_grad_checkpointing() diff --git a/src/training/params.py b/src/training/params.py index a0640f456..83893d020 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -382,13 +382,19 @@ def parse_args(args): "--lock-text-unlocked-layers", type=int, default=0, - help="Leave last n image tower layer groups unlocked.", + help="Leave last n text tower layer groups unlocked.", + ) + parser.add_argument( + "--text-unlocked-biases", + type=bool, + default=False, + help="Only unlock model biases", ) parser.add_argument( "--lock-text-freeze-layer-norm", default=False, action='store_true', - help="Freeze BatchNorm running stats in image tower for any locked layers.", + help="Freeze BatchNorm running stats in text tower for any locked layers.", ) parser.add_argument( "--lock-tower-b", @@ -400,7 +406,7 @@ def parse_args(args): "--lock-tower-b-unlocked-layers", type=int, default=0, - help="Leave last n image tower layer groups unlocked.", + help="Leave last n tower layer groups unlocked.", ) parser.add_argument( "--lock-tower-b-freeze-layer-norm", @@ -418,7 +424,7 @@ def parse_args(args): "--lock-tower-a-unlocked-layers", type=int, default=0, - help="Leave last n image tower layer groups unlocked.", + help="Leave last n tower layer groups unlocked.", ) parser.add_argument( "--lock-tower-a-freeze-layer-norm", @@ -426,6 +432,18 @@ def parse_args(args): action='store_true', help="Freeze BatchNorm running stats in image tower for any locked layers.", ) + parser.add_argument( + "--text-b-unlocked-biases", + type=bool, + default=False, + help="Only unlock model biases", + ) + parser.add_argument( + "--text-a-unlocked-biases", + type=bool, + default=False, + help="Only unlock model biases", + ) parser.add_argument( "--log-every-n-steps", type=int, diff --git a/tests/test_TextTextCLIP.py b/tests/test_TextTextCLIP.py index d9df60422..bfbbdc51f 100644 --- a/tests/test_TextTextCLIP.py +++ b/tests/test_TextTextCLIP.py @@ -28,6 +28,10 @@ def test_inference_simple(model_cfg, pretrained=None): text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) print(text_probs) + + model.lock_tower_a(unlocked_layers=0,freeze_layer_norm=True,unlock_biases=True) + model.lock_tower_b(unlocked_layers=0,freeze_layer_norm=True,unlock_biases=True) + @pytest.mark.parametrize("model_cfg", [("Siamese-xlm-roberta-large")]) @@ -47,9 +51,11 @@ def test_inference_simple(model_cfg, pretrained=None): text_probs = (100.0 * text_a_features @ text_b_features.T).softmax(dim=-1) print(text_probs) + + model.lock_text_tower(unlocked_layers=0,freeze_layer_norm=True,unlock_biases=True) -@pytest.mark.parametrize("model_cfg", [("pythia-1b-pythia-1b")]) +@pytest.mark.parametrize("model_cfg", [("pythia-410m-pythia-410m")]) def test_inference_simple(model_cfg, pretrained=None): model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='text_dual_encoder') tokenizer = get_tokenizer(model_cfg) From 4f89a44109bfd870932ffb60e307450501bd815d Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Sun, 2 Apr 2023 19:48:52 -0400 Subject: [PATCH 29/36] fixed a typo --- .../xlm-roberta-large-xlm-roberta-large.json | 16 ++++++++++++++++ src/training/main.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json diff --git a/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json b/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json new file mode 100644 index 000000000..766e71439 --- /dev/null +++ b/src/open_clip/model_configs/xlm-roberta-large-xlm-roberta-large.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "tower_a_cfg": { + "hf_model_name": "xlm-roberta-large", + "hf_tokenizer_name": "xlm-roberta-large", + "proj": "mlp", + "pooler_type": "mean_pooler" + }, + "tower_b_cfg": { + "hf_model_name": "xlm-roberta-large", + "hf_tokenizer_name": "xlm-roberta-large", + "proj": "mlp", + "pooler_type": "mean_pooler" + } +} \ No newline at end of file diff --git a/src/training/main.py b/src/training/main.py index 19dbea851..54ff04072 100644 --- a/src/training/main.py +++ b/src/training/main.py @@ -256,7 +256,7 @@ def main(args): model.lock_text_tower( unlocked_layers=args.lock_text_unlocked_layers, freeze_layer_norm=args.lock_text_freeze_layer_norm, - unlocked_biases=args.text_unlocked_biase) + unlocked_biases=args.text_unlocked_biases) if args.lock_tower_a: model.lock_tower_a( unlocked_layers=args.lock_tower_a_unlocked_layers, From 3dddc3fe87213858d5ca801401f29473256afd84 Mon Sep 17 00:00:00 2001 From: lingjzhu <> Date: Sun, 30 Apr 2023 16:05:43 -0700 Subject: [PATCH 30/36] add contriever training --- src/open_clip/factory.py | 4 +- src/open_clip/model.py | 4 +- src/open_clip/text_augmentation.py | 107 ++++++++++++++++++ src/training/data.py | 14 ++- src/training/params.py | 41 ++++++- tests/{test_TextTextCLIP.py => test_CLANP.py} | 8 +- 6 files changed, 165 insertions(+), 13 deletions(-) create mode 100644 src/open_clip/text_augmentation.py rename tests/{test_TextTextCLIP.py => test_CLANP.py} (90%) diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 630adcfb4..4458b5b8e 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -205,14 +205,14 @@ def create_model( custom_text = model_cfg.pop('custom_text', False) or force_custom_text or is_hf_model # switch to TextTextCLIP - if model_type=="text_dual_encoder": + if model_type=="CLANP": if 'hf_model_name' in model_cfg.get('tower_a_cfg', {}): model_cfg['tower_a_cfg']['hf_model_pretrained'] = pretrained_hf if 'hf_model_name' in model_cfg.get('tower_b_cfg', {}): model_cfg['tower_b_cfg']['hf_model_pretrained'] = pretrained_hf model = TextTextCLIP(**model_cfg, cast_dtype=cast_dtype) - elif model_type=="text_siamese_encoder": + elif model_type=="SiameseCLANP": if 'hf_model_name' in model_cfg.get('tower_a_cfg', {}): model_cfg['tower_a_cfg']['hf_model_pretrained'] = pretrained_hf model = SiameseTextCLIP(**model_cfg, cast_dtype=cast_dtype) diff --git a/src/open_clip/model.py b/src/open_clip/model.py index 998dd6c11..0b0b97376 100644 --- a/src/open_clip/model.py +++ b/src/open_clip/model.py @@ -290,7 +290,7 @@ def forward(self, image, text): -class TextTextCLIP(nn.Module): +class CLANP(nn.Module): def __init__( self, embed_dim: int, @@ -339,7 +339,7 @@ def forward(self, text_a, text_b): -class SiameseTextCLIP(nn.Module): +class SiameseCLANP(nn.Module): def __init__( self, embed_dim: int, diff --git a/src/open_clip/text_augmentation.py b/src/open_clip/text_augmentation.py new file mode 100644 index 000000000..9369fe4c7 --- /dev/null +++ b/src/open_clip/text_augmentation.py @@ -0,0 +1,107 @@ +import torch +import random +import numpy as np + + +''' +Data augmentation scripts from the original contriever implementation: https://github.com/facebookresearch/contriever/blob/main/src/data.py +''' + +def randomcrop(x, ratio_min, ratio_max): + + ratio = random.uniform(ratio_min, ratio_max) + length = int(len(x) * ratio) + start = random.randint(0, len(x) - length) + end = start + length + crop = x[start:end].clone() + return crop + + +def add_token(x, token): + x = torch.cat((torch.tensor([token]), x)) + return x + + +def deleteword(x, p=0.1): + mask = np.random.rand(len(x)) + x = [e for e, m in zip(x, mask) if m > p] + return x + + +def replaceword(x, min_random, max_random, p=0.1): + mask = np.random.rand(len(x)) + x = [e if m > p else random.randint(min_random, max_random) for e, m in zip(x, mask)] + return x + + +def maskword(x, mask_id, p=0.1): + mask = np.random.rand(len(x)) + x = [e if m > p else mask_id for e, m in zip(x, mask)] + return x + + +def shuffleword(x, p=0.1): + count = (np.random.rand(len(x)) < p).sum() + """Shuffles any n number of values in a list""" + indices_to_shuffle = random.sample(range(len(x)), k=count) + to_shuffle = [x[i] for i in indices_to_shuffle] + random.shuffle(to_shuffle) + for index, value in enumerate(to_shuffle): + old_index = indices_to_shuffle[index] + x[old_index] = value + return x + + +def apply_augmentation(x, tokenizer,args): + if args.text_augmentation == "mask": + return torch.tensor(maskword(x, mask_id=tokenizer.mask_token_id, p=args.text_prob_augmentation)) + elif args.text_augmentation == "replace": + return torch.tensor( + replaceword(x, min_random=args.start_id, max_random=len(tokenizer) - 1, p=args.text_prob_augmentation) + ) + elif args.text_augmentation == "delete": + return torch.tensor(deleteword(x, p=args.text_prob_augmentation)) + elif args.text_augmentation == "shuffle": + return torch.tensor(shuffleword(x, p=args.text_prob_augmentation)) + else: + if not isinstance(x, torch.Tensor): + x = torch.Tensor(x) + return x + + +def add_bos_eos(x, bos_token_id, eos_token_id): + if not isinstance(x, torch.Tensor): + x = torch.Tensor(x) + if bos_token_id is None and eos_token_id is not None: + x = torch.cat([x.clone().detach(), torch.tensor([eos_token_id])]) + elif bos_token_id is not None and eos_token_id is None: + x = torch.cat([torch.tensor([bos_token_id]), x.clone().detach()]) + elif bos_token_id is None and eos_token_id is None: + pass + else: + x = torch.cat([torch.tensor([bos_token_id]), x.clone().detach(), torch.tensor([eos_token_id])]) + return x + + +def create_unsupervised_sample(text,tokenizer,args): + # remove padding + text = text[text!=tokenizer.pad_token_id] + if text[0] == tokenizer.bos_token_id: + text == text[1:] + if text[-1] == tokenizer.eos_token_id: + text = text[:-1] + # perform data augmentation + text = randomcrop(text, args.text_aug_ratio_min, args.text_aug_ratio_max) + text = apply_augmentation(text, tokenizer, args) + # add bos/eos tokens back + text = add_bos_eos(text, tokenizer.bos_token_id, tokenizer.eos_token_id) + # padding back to max length + text = tokenizer( + text, + return_tensors='pt', + max_length=args.context_length, + padding='max_length', + truncation=True, + ).input_ids + + return text \ No newline at end of file diff --git a/src/training/data.py b/src/training/data.py index 256472896..688014050 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -21,6 +21,7 @@ from torch.utils.data.distributed import DistributedSampler from webdataset.filters import _shuffle from webdataset.tariterators import base_plus_ext, url_opener, tar_file_expander, valid_sample +from open_clip.text_augmentation import create_unsupervised_sample try: import horovod.torch as hvd @@ -438,14 +439,19 @@ def get_wds_dataset(args, preprocess_img, is_train, epoch=0, floor=False, tokeni wds.tarfile_to_samples(handler=log_and_continue), ]) - if 'text' in args.model_type: + if 'CLANP' in args.model_type and not args.unsupervised_pretraining: pipeline.extend([ - wds.to_tuple("text_a", "text_b"), + wds.to_tuple(args.text_a_key, args.text_b_key), wds.map_tuple(lambda text: tokenizer(text.decode('utf8'))[0], lambda text: tokenizer(text.decode('utf8'))[0]), wds.batched(args.batch_size, partial=not is_train), ]) - - + elif 'CLANP' in args.model_type and args.unsupervised_pretraining: + pipeline.extend([ + wds.to_tuple(args.text_a_key, args.text_b_key), + wds.map_tuple(lambda text: tokenizer(text.decode('utf8'))[0], lambda text: tokenizer(text.decode('utf8'))[0]), + wds.map_tuple(lambda text: create_unsupervised_sample(text,tokenizer,args), lambda text: create_unsupervised_sample(text,tokenizer,args)), + wds.batched(args.batch_size, partial=not is_train), + ]) else: pipeline.extend([ wds.select(filter_no_caption_or_no_image), diff --git a/src/training/params.py b/src/training/params.py index 83893d020..caf6825ac 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -453,7 +453,7 @@ def parse_args(args): parser.add_argument( "--model-type", type=str, - choices=["CLIP", "text_dual_encoder", "text_siamese_encoder"], + choices=["CLIP", "CLANP", "SiameseCLANP"], default="CLIP", help="Specify the model type", ) @@ -505,6 +505,45 @@ def parse_args(args): default=None, help='Which pre-trained weights to distill from, if any.' ) + + ''' + Arguments related to training a contriever-like text-to-text model + ''' + parser.add_argument( + "--unsupervised-pretraining", + action="store_true", + help="Whether to run contriever-style unupservised contrastive pretraining" + ) + parser.add_argument( + "--text-augmentation", + default='delete', + choices=['delete','mask','replace','shuffle'], + help='Which text augmentation to create postive pairs for training unsupervised text retreival models' + ) + parser.add_argument( + "--text-aug-ratio-min", + default=0.1, + type=float, + help='min ratio for random cropping of texts (only applies to unsupervised text retreival models)' + ) + parser.add_argument( + "--text-aug-ratio-max", + default=0.5, + type=float, + help='max ratio for random cropping of texts (only applies to unsupervised text retreival models)' + ) + parser.add_argument( + "--text-prob-augmentation", + default=0.1, + type=float, + help='probability of applying augmentations to texts (only applies to unsupervised text retreival models)' + ) + parser.add_argument( + "--context-length", + default=77, + type=int, + help="max number of tokens in the input text" + ) args = parser.parse_args(args) # If some params are not passed, we use the default values based on model name. diff --git a/tests/test_TextTextCLIP.py b/tests/test_CLANP.py similarity index 90% rename from tests/test_TextTextCLIP.py rename to tests/test_CLANP.py index bfbbdc51f..b198d9e56 100644 --- a/tests/test_TextTextCLIP.py +++ b/tests/test_CLANP.py @@ -13,7 +13,7 @@ @pytest.mark.parametrize("model_cfg", [("roberta-roberta")]) def test_inference_simple(model_cfg, pretrained=None): - model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='text_dual_encoder') + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='CLANP') tokenizer = get_tokenizer(model_cfg) text_a = tokenizer(['this', 'is', 'a', 'document']) @@ -36,7 +36,7 @@ def test_inference_simple(model_cfg, pretrained=None): @pytest.mark.parametrize("model_cfg", [("Siamese-xlm-roberta-large")]) def test_inference_simple(model_cfg, pretrained=None): - model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='text_siamese_encoder') + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='SiameseCLANP') tokenizer = get_tokenizer(model_cfg) text_a = tokenizer(['this', 'is', 'a', 'document']) @@ -57,7 +57,7 @@ def test_inference_simple(model_cfg, pretrained=None): @pytest.mark.parametrize("model_cfg", [("pythia-410m-pythia-410m")]) def test_inference_simple(model_cfg, pretrained=None): - model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='text_dual_encoder') + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='CLANP') tokenizer = get_tokenizer(model_cfg) text_a = tokenizer(['this', 'is', 'a', 'document']) @@ -76,7 +76,7 @@ def test_inference_simple(model_cfg, pretrained=None): @pytest.mark.parametrize("model_cfg", [("Siamese-pythia-410m")]) def test_inference_simple(model_cfg, pretrained=None): - model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='text_siamese_encoder') + model, _, preprocess = open_clip.create_model_and_transforms(model_cfg, pretrained=pretrained, jit=False, model_type='SiameseCLANP') tokenizer = get_tokenizer(model_cfg) text_a = tokenizer(['this', 'is', 'a', 'document']) From ab6c0b5bb51c763de6ac94a0c6c847ca3bfced3b Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Wed, 10 May 2023 10:50:20 -0400 Subject: [PATCH 31/36] fix import --- docs/script_examples/text_example.sh | 2 +- src/open_clip/__init__.py | 2 +- src/open_clip/factory.py | 8 +++--- .../model_configs/pythia-70m-pythia-70m.json | 16 +++++++++++ src/open_clip/text_augmentation.py | 22 ++++++--------- train_clanp.sh | 28 +++++++++++++++++++ 6 files changed, 58 insertions(+), 20 deletions(-) create mode 100644 src/open_clip/model_configs/pythia-70m-pythia-70m.json create mode 100644 train_clanp.sh diff --git a/docs/script_examples/text_example.sh b/docs/script_examples/text_example.sh index faba97f30..7d1a16d3e 100644 --- a/docs/script_examples/text_example.sh +++ b/docs/script_examples/text_example.sh @@ -53,7 +53,7 @@ srun --comment laion --cpu_bind=v --accel-bind=gn python3 -m training.main \ --gather-with-grad \ --ddp-static-graph \ --grad-checkpointing \ - --model_type="text_siamese_encoder" \ + --model_type="Siamese_CLANP" \ --debug \ --sts-val-data="lingjzhu/sts17-crosslingual" diff --git a/src/open_clip/__init__.py b/src/open_clip/__init__.py index 01fd7a42b..30f59fb80 100644 --- a/src/open_clip/__init__.py +++ b/src/open_clip/__init__.py @@ -4,7 +4,7 @@ from .factory import list_models, add_model_config, get_model_config, load_checkpoint from .loss import ClipLoss, DistillClipLoss, CoCaLoss -from .model import CLIP, CustomTextCLIP, TextTextCLIP, SiameseTextCLIP, CLIPTextCfg, CLIPVisionCfg,\ +from .model import CLIP, CustomTextCLIP, SiameseCLANP, CLANP, CLIPTextCfg, CLIPVisionCfg,\ convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype from .openai import load_openai_model, list_openai_models from .pretrained import list_pretrained, list_pretrained_models_by_tag, list_pretrained_tags_by_model, \ diff --git a/src/open_clip/factory.py b/src/open_clip/factory.py index 4458b5b8e..92be625ff 100644 --- a/src/open_clip/factory.py +++ b/src/open_clip/factory.py @@ -10,7 +10,7 @@ import torch from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD -from .model import CLIP, CustomTextCLIP, TextTextCLIP, SiameseTextCLIP, convert_weights_to_lp, convert_to_custom_text_state_dict,\ +from .model import CLIP, CustomTextCLIP, CLANP, SiameseCLANP, convert_weights_to_lp, convert_to_custom_text_state_dict,\ resize_pos_embed, get_cast_dtype from .coca_model import CoCa from .loss import ClipLoss, DistillClipLoss, CoCaLoss @@ -204,18 +204,18 @@ def create_model( is_hf_model = 'hf_model_name' in model_cfg.get('text_cfg', {}) custom_text = model_cfg.pop('custom_text', False) or force_custom_text or is_hf_model - # switch to TextTextCLIP + # switch to CLANP if model_type=="CLANP": if 'hf_model_name' in model_cfg.get('tower_a_cfg', {}): model_cfg['tower_a_cfg']['hf_model_pretrained'] = pretrained_hf if 'hf_model_name' in model_cfg.get('tower_b_cfg', {}): model_cfg['tower_b_cfg']['hf_model_pretrained'] = pretrained_hf - model = TextTextCLIP(**model_cfg, cast_dtype=cast_dtype) + model = CLANP(**model_cfg, cast_dtype=cast_dtype) elif model_type=="SiameseCLANP": if 'hf_model_name' in model_cfg.get('tower_a_cfg', {}): model_cfg['tower_a_cfg']['hf_model_pretrained'] = pretrained_hf - model = SiameseTextCLIP(**model_cfg, cast_dtype=cast_dtype) + model = SiameseCLANP(**model_cfg, cast_dtype=cast_dtype) elif model_type=="CLIP": diff --git a/src/open_clip/model_configs/pythia-70m-pythia-70m.json b/src/open_clip/model_configs/pythia-70m-pythia-70m.json new file mode 100644 index 000000000..5c49ed5c5 --- /dev/null +++ b/src/open_clip/model_configs/pythia-70m-pythia-70m.json @@ -0,0 +1,16 @@ +{ + "embed_dim": 512, + "quick_gelu": true, + "tower_a_cfg": { + "hf_model_name": "EleutherAI/pythia-70m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-70m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + }, + "tower_b_cfg": { + "hf_model_name": "EleutherAI/pythia-70m-deduped", + "hf_tokenizer_name": "EleutherAI/pythia-70m-deduped", + "proj": "mlp", + "pooler_type": "weighted_mean_pooler" + } +} \ No newline at end of file diff --git a/src/open_clip/text_augmentation.py b/src/open_clip/text_augmentation.py index 9369fe4c7..e98cc0948 100644 --- a/src/open_clip/text_augmentation.py +++ b/src/open_clip/text_augmentation.py @@ -85,23 +85,17 @@ def add_bos_eos(x, bos_token_id, eos_token_id): def create_unsupervised_sample(text,tokenizer,args): # remove padding - text = text[text!=tokenizer.pad_token_id] - if text[0] == tokenizer.bos_token_id: + text = text[text!=tokenizer.tokenizer.pad_token_id] + if text[0] == tokenizer.tokenizer.bos_token_id: text == text[1:] - if text[-1] == tokenizer.eos_token_id: + if text[-1] == tokenizer.tokenizer.eos_token_id: text = text[:-1] # perform data augmentation text = randomcrop(text, args.text_aug_ratio_min, args.text_aug_ratio_max) - text = apply_augmentation(text, tokenizer, args) + text = apply_augmentation(text, tokenizer.tokenizer, args) # add bos/eos tokens back - text = add_bos_eos(text, tokenizer.bos_token_id, tokenizer.eos_token_id) + text = add_bos_eos(text, tokenizer.tokenizer.bos_token_id, tokenizer.tokenizer.eos_token_id) # padding back to max length - text = tokenizer( - text, - return_tensors='pt', - max_length=args.context_length, - padding='max_length', - truncation=True, - ).input_ids - - return text \ No newline at end of file + output = torch.zeros(args.context_length) + output[:len(text)] = text + return output.long() \ No newline at end of file diff --git a/train_clanp.sh b/train_clanp.sh new file mode 100644 index 000000000..f8d1cdcf4 --- /dev/null +++ b/train_clanp.sh @@ -0,0 +1,28 @@ +torchrun --standalone --nnodes=1 src/training/main.py \ + --save-frequency 1 \ + --dataset-type="webdataset" \ + --text-a-key="txt" \ + --text-b-key="txt" \ + --report-to wandb \ + --wandb-project-name="TextTextCLIP" \ + --train-data="../text.tar" \ + --train-num-samples 13560 \ + --warmup 2000 \ + --batch-size=4 \ + --precision amp_bfloat16 \ + --lr=0.001 \ + --wd=0.2 \ + --epochs=9 \ + --workers=1 \ + --model "pythia-70m-pythia-70m" \ + --seed 0 \ + --log-every-n-steps 5 \ + --model-type "CLANP" \ + --sts-val-data "lingjzhu/sts17-crosslingual" \ + --local-loss \ + --gather-with-grad \ + --ddp-static-graph \ + --grad-checkpointing \ + --debug \ + --unsupervised-pretraining + From edef6736b870b0d6c2c64bb1a2a075513d3b0f58 Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Wed, 10 May 2023 10:50:57 -0400 Subject: [PATCH 32/36] fix import --- train_clanp.sh => docs/script_examples/train_clanp.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename train_clanp.sh => docs/script_examples/train_clanp.sh (100%) diff --git a/train_clanp.sh b/docs/script_examples/train_clanp.sh similarity index 100% rename from train_clanp.sh rename to docs/script_examples/train_clanp.sh From a495a8a27a82f804d33add3a71158ccf94f4ea7f Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Sun, 21 May 2023 18:16:36 +0000 Subject: [PATCH 33/36] fix agumentation script --- docs/script_examples/text_example.sh | 59 ---------------------------- src/open_clip/text_augmentation.py | 8 ++-- src/training/data.py | 4 +- src/training/params.py | 6 +-- 4 files changed, 9 insertions(+), 68 deletions(-) delete mode 100644 docs/script_examples/text_example.sh diff --git a/docs/script_examples/text_example.sh b/docs/script_examples/text_example.sh deleted file mode 100644 index 7d1a16d3e..000000000 --- a/docs/script_examples/text_example.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -#SBATCH --partition=g80 -#SBATCH --account=laion -#SBATCH --job-name=TextTextCLIP -#SBATCH --nodes=32 -#SBATCH --ntasks-per-node=4 -#SBATCH --cpus-per-task=4 -#SBATCH --output=logs/%x_%j.out -#SBATCH --comment=laion -#SBATCH --open-mode=append -#SBATCH --exclusive - -module load openmpi -module load cuda/11.7 - -export PYTHONFAULTHANDLER=1 -export CUDA_LAUNCH_BLOCKING=0 -export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"` -export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1) -export MASTER_PORT=12802 -export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l` - -echo go $COUNT_NODE -echo $HOSTNAMES - - -cd /admin/home-jianz/open_clip/src -export PYTHONPATH="$PYTHONPATH:/admin/home-jianz/open_clip/src" - -EXP_NAME="" - -#srun --comment laion --cpu_bind=v --accel-bind=gn torchrun --nproc_per_node 4 --max_restarts=3 --rdzv_id=1 --rdzv_backend=c10d --rdzv_endpoint=localhost:0 -m training.main \ -srun --comment laion --cpu_bind=v --accel-bind=gn python3 -m training.main \ - --save-frequency 1 \ - --dataset-type="webdataset" \ - --text-a-key="text_a" \ - --text-b-key="text_b" \ - --report-to wandb \ - --wandb-project-name="TextTextCLIP" \ - --train-data="/fsx/home-jianz/mycache/huggingface/hub/datasets--lingjzhu--laion-multi-2B/snapshots/7ec0d572ac4d8da1e6997ed32e383ab63967e05d/laion-multi-2B-{000..127}.tar" \ - --train-num-samples 135646078 \ - --warmup 2000 \ - --batch-size=2048 \ - --precision amp_bfloat16 \ - --lr=0.001 \ - --wd=0.2 \ - --epochs=97 \ - --workers=1 \ - --model="Siamese-xlm-roberta-large" \ - --seed 0 \ - --log-every-n-steps 5 \ - --local-loss \ - --gather-with-grad \ - --ddp-static-graph \ - --grad-checkpointing \ - --model_type="Siamese_CLANP" \ - --debug \ - --sts-val-data="lingjzhu/sts17-crosslingual" - diff --git a/src/open_clip/text_augmentation.py b/src/open_clip/text_augmentation.py index e98cc0948..66bd8a568 100644 --- a/src/open_clip/text_augmentation.py +++ b/src/open_clip/text_augmentation.py @@ -86,10 +86,10 @@ def add_bos_eos(x, bos_token_id, eos_token_id): def create_unsupervised_sample(text,tokenizer,args): # remove padding text = text[text!=tokenizer.tokenizer.pad_token_id] - if text[0] == tokenizer.tokenizer.bos_token_id: - text == text[1:] - if text[-1] == tokenizer.tokenizer.eos_token_id: - text = text[:-1] + if tokenizer.tokenizer.bos_token_id is not None: + text = text[text!=tokenizer.tokenizer.bos_token_id] + if tokenizer.tokenizer.eos_token_id is not None: + text = text[text!=tokenizer.tokenizer.eos_token_id] # perform data augmentation text = randomcrop(text, args.text_aug_ratio_min, args.text_aug_ratio_max) text = apply_augmentation(text, tokenizer.tokenizer, args) diff --git a/src/training/data.py b/src/training/data.py index 688014050..8bee63eb3 100644 --- a/src/training/data.py +++ b/src/training/data.py @@ -442,13 +442,13 @@ def get_wds_dataset(args, preprocess_img, is_train, epoch=0, floor=False, tokeni if 'CLANP' in args.model_type and not args.unsupervised_pretraining: pipeline.extend([ wds.to_tuple(args.text_a_key, args.text_b_key), - wds.map_tuple(lambda text: tokenizer(text.decode('utf8'))[0], lambda text: tokenizer(text.decode('utf8'))[0]), + wds.map_tuple(lambda text: tokenizer(text.decode('utf8'),context_length=args.context_length)[0], lambda text: tokenizer(text.decode('utf8'),context_length=args.context_length)[0]), wds.batched(args.batch_size, partial=not is_train), ]) elif 'CLANP' in args.model_type and args.unsupervised_pretraining: pipeline.extend([ wds.to_tuple(args.text_a_key, args.text_b_key), - wds.map_tuple(lambda text: tokenizer(text.decode('utf8'))[0], lambda text: tokenizer(text.decode('utf8'))[0]), + wds.map_tuple(lambda text: tokenizer(text.decode('utf8'),context_length=args.context_length)[0], lambda text: tokenizer(text.decode('utf8'),context_length=args.context_length)[0]), wds.map_tuple(lambda text: create_unsupervised_sample(text,tokenizer,args), lambda text: create_unsupervised_sample(text,tokenizer,args)), wds.batched(args.batch_size, partial=not is_train), ]) diff --git a/src/training/params.py b/src/training/params.py index caf6825ac..f5f8f39ea 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -386,7 +386,7 @@ def parse_args(args): ) parser.add_argument( "--text-unlocked-biases", - type=bool, + action='store_true', default=False, help="Only unlock model biases", ) @@ -434,13 +434,13 @@ def parse_args(args): ) parser.add_argument( "--text-b-unlocked-biases", - type=bool, + action='store_true', default=False, help="Only unlock model biases", ) parser.add_argument( "--text-a-unlocked-biases", - type=bool, + action='store_true', default=False, help="Only unlock model biases", ) From f7c7f19e17f96e381737b922167a1cbaa6a461a1 Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Mon, 22 May 2023 04:57:12 +0000 Subject: [PATCH 34/36] add MTEB evaluation --- error_logs.txt | 2987 +++++++++++++++++++++++++++++++++++++ src/training/mteb_eval.py | 62 + src/training/params.py | 6 + 3 files changed, 3055 insertions(+) create mode 100644 error_logs.txt create mode 100644 src/training/mteb_eval.py diff --git a/error_logs.txt b/error_logs.txt new file mode 100644 index 000000000..8df5d2ad3 --- /dev/null +++ b/error_logs.txt @@ -0,0 +1,2987 @@ +2023-05-21 22:14:41.116492 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:43.063198 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:46.659623 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:47.605287 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:49.655345 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:50.423953 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:14:51.993315 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:01.074955 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:03.724835 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:06.890844 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:07.588629 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 35, in encode + features = self.tokenize(sentences_batch) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1269, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'CLANP_For_MTEB' object has no attribute 'tokenize' + + +2023-05-21 22:15:07.592022 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.595497 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.597198 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.598847 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.600403 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.602096 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.603802 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.605398 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.606962 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.608499 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.610021 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.611630 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.613429 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.614980 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.616518 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.618017 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.619576 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.621133 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.622719 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.624253 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.625774 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.627625 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.629457 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.631024 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.632535 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.634130 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:15:07.635689 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:07.819906 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1673 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:08.270324 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 25074 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:08.768140 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 380 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:09.064860 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 28521 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:09.547762 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 998 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:09.830162 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 998 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:10.178599 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 35 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:10.718680 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 56 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:11.112059 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 2513 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:11.606707 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1694 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:12.282844 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 46 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:18:12.285650 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.288074 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.289633 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.291188 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.292730 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.294196 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.295704 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.297226 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.298715 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.300210 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.301720 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.303295 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.304907 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.306460 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.307999 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.309583 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.311148 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.312693 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.314289 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.315852 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.317538 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.319044 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.320612 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.322100 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.323566 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.325068 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:18:12.326566 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:13.233910 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:13.672080 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:14.092847 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:14.347831 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:14.747403 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:14.992638 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:15.311379 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:15.762837 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:16.138870 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:16.554528 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:16.769926 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 34, in encode + print(sentences_batch.shape) +AttributeError: 'list' object has no attribute 'shape' + + +2023-05-21 22:26:16.772275 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.775636 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.777437 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.778978 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.780513 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.782036 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.783554 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.785117 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.786643 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.788211 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.789983 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.791514 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.793020 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.794537 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.796097 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.797619 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.799175 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.800729 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.802244 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.803751 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.805325 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.806855 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.808415 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.809923 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.811601 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.813134 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:26:16.814639 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:22.007826 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1673 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:22.261201 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 25074 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:22.544696 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 380 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:23.258645 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 28521 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:23.632457 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 998 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:23.983727 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 998 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:24.197034 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 35 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:24.545544 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1620 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:24.917339 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 2513 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:25.195054 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 1694 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:25.435361 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 36, in encode + features = self.batch_to_device(features, self.device) + File "src/training/mteb_eval.py", line 53, in batch_to_device + if isinstance(batch[key], Tensor): +IndexError: index 46 is out of bounds for dimension 0 with size 32 + + +2023-05-21 22:27:25.437428 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.439691 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.440930 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.442174 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.443383 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.444590 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.445818 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.447101 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.448322 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.449536 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.450682 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.451912 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.453140 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.454311 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.455513 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.456750 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.457911 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.459269 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.460568 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.461740 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.462949 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.464165 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.465347 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.466498 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.467773 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.469019 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:27:25.470211 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:19.439473 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:19.700152 >>> ArxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:19.977665 >>> BiorxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:20.230267 >>> BiorxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:20.733546 >>> MedrxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:20.977127 >>> MedrxivClusteringS2S +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:21.200248 >>> RedditClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:21.507616 >>> RedditClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:21.764115 >>> StackExchangeClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:22.154970 >>> StackExchangeClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:22.423249 >>> TwentyNewsgroupsClustering +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "src/training/mteb_eval.py", line 37, in encode + text_features = model.encode_text(features) +NameError: name 'model' is not defined + + +2023-05-21 22:28:22.425438 >>> ArguAna +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.427412 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.428703 >>> CQADupstackAndroidRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.430040 >>> CQADupstackEnglishRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.431280 >>> CQADupstackGamingRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.432470 >>> CQADupstackGisRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.433670 >>> CQADupstackMathematicaRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.434858 >>> CQADupstackPhysicsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.436104 >>> CQADupstackProgrammersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.437305 >>> CQADupstackStatsRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.438490 >>> CQADupstackTexRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.439876 >>> CQADupstackUnixRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.441061 >>> CQADupstackWebmastersRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.442262 >>> CQADupstackWordpressRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.443481 >>> DBPedia +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.444719 >>> FEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.445916 >>> FiQA2018 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.447140 >>> HotpotQA +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.448406 >>> MSMARCO +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.449612 >>> MSMARCOv2 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.450877 >>> NFCorpus +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.452045 >>> NQ +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.453390 >>> QuoraRetrieval +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.454588 >>> SCIDOCS +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.455874 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.457095 >>> Touche2020 +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:28:22.458297 >>> TRECCOVID +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 16, in load_data + from beir import util +ModuleNotFoundError: No module named 'beir' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 225, in run + task.load_data(eval_splits=task_eval_splits) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/BeIRTask.py", line 18, in load_data + raise Exception("Retrieval tasks require beir package. Please install it with `pip install mteb[beir]`") +Exception: Retrieval tasks require beir package. Please install it with `pip install mteb[beir]` + + +2023-05-21 22:30:44.495512 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/torch/_tensor.py", line 956, in __array__ + return self.numpy() +TypeError: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first. + + +2023-05-21 22:33:15.493726 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) +ValueError: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (8,) + inhomogeneous part. + + +2023-05-21 22:36:46.248030 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) +ValueError: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (25000,) + inhomogeneous part. + + +2023-05-21 22:42:50.337546 >>> ArxivClusteringP2P +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskClustering.py", line 19, in evaluate + metrics = evaluator(model) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/evaluators/ClusteringEvaluator.py", line 25, in __call__ + corpus_embeddings = np.asarray(model.encode(self.sentences)) +ValueError: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (25000,) + inhomogeneous part. + + diff --git a/src/training/mteb_eval.py b/src/training/mteb_eval.py new file mode 100644 index 000000000..241713bc4 --- /dev/null +++ b/src/training/mteb_eval.py @@ -0,0 +1,62 @@ +import sys +import torch +import torch.nn as nn +from mteb import MTEB +from training.params import parse_args +from tqdm.autonotebook import trange +import open_clip +from open_clip.factory import get_tokenizer + +class CLANP_For_MTEB(nn.Module): + + def __init__(self,args): + super().__init__() + self.model, _, _ = open_clip.create_model_and_transforms(args.model, pretrained=args.pretrained, jit=False, model_type='SiameseCLANP') + self.tokenizer = get_tokenizer(args.model) + self.model.eval() + self.device = 'cuda' if torch.cuda.is_available() else 'cpu' + self.model.to(self.device) + + def encode(self, sentences, batch_size=32, show_progress_bar=True, **kwargs): + """ Returns a list of embeddings for the given sentences. + Args: + sentences (`List[str]`): List of sentences to encode + batch_size (`int`): Batch size for the encoding + + Returns: + `List[np.ndarray]` or `List[tensor]`: List of embeddings for the given sentences + """ + + + all_embeddings = [] + for start_index in trange(0, len(sentences), batch_size, desc="Batches", disable=not show_progress_bar): + sentences_batch = sentences[start_index:start_index+batch_size] + features = self.tokenizer(sentences_batch) + features = features.to(self.device) + with torch.no_grad(): + text_features = self.model.encode_text(features) + embeddings = [f for f in text_features.detach().cpu().numpy()] + + all_embeddings.extend(embeddings) + + return all_embeddings + + + + +def run_mteb(args): + + model = CLANP_For_MTEB(args) + evaluation = MTEB(tasks=['STS22']) #task_types=['Clustering','retrieval'] + results = evaluation.run(model, eval_splits=["test"], output_folder=args.mteb_output) + + +if __name__ == "__main__": + + args = parse_args(sys.argv[1:]) + run_mteb(args) + + # /fsx/home-jianz/logs/2023_05_11-16_04_50-model_Siamese-pythia-410m-lr_0.0001-b_1024-j_6-p_amp_bfloat16/checkpoints/epoch_1.pt + # srun --account=laion --partition=g40 nodes=1 --ntasks-per-node=1 --cpus-per-task=4 --job-name=MTEB --pty bash -i + # python3 src/training/mteb_eval.py --model="Siamese-pythia-410m" --pretrained /fsx/home-jianz/logs/2023_05_11-16_04_50-model_Siamese-pythia-410m-lr_0.0001-b_1024-j_6-p_amp_bfloat16/checkpoints/epoch_1.pt --mteb-output /fsx/home-jianz/logs/mteb --context-length=77 + \ No newline at end of file diff --git a/src/training/params.py b/src/training/params.py index f5f8f39ea..b4da2cdc3 100644 --- a/src/training/params.py +++ b/src/training/params.py @@ -544,6 +544,12 @@ def parse_args(args): type=int, help="max number of tokens in the input text" ) + parser.add_argument( + "--mteb-output", + default=None, + type=str, + help="location to store MTEB evaluation results" + ) args = parser.parse_args(args) # If some params are not passed, we use the default values based on model name. From 330d4d9e70867e1e398db6782c6c3927757545bd Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Tue, 20 Jun 2023 05:26:32 +0000 Subject: [PATCH 35/36] MTEB benchmark --- error_logs.txt | 129 ++++++++++++++++++ .../model_configs/Siamese-mt5-xl.json | 10 ++ src/training/mteb_eval.py | 15 +- 3 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 src/open_clip/model_configs/Siamese-mt5-xl.json diff --git a/error_logs.txt b/error_logs.txt index 8df5d2ad3..1bd406aa9 100644 --- a/error_logs.txt +++ b/error_logs.txt @@ -2985,3 +2985,132 @@ Traceback (most recent call last): ValueError: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (25000,) + inhomogeneous part. +2023-05-29 16:35:30.769925 >>> ClimateFEVER +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 39, in search + query_embeddings = self.model.encode_queries( + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 148, in encode_queries + return self.model.encode(queries, batch_size=batch_size, **kwargs) + File "src/training/mteb_eval.py", line 42, in encode + return np.stack(all_embeddings,0) +NameError: name 'np' is not defined + + +2023-05-29 17:10:01.510309 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 24, in cos_sim + return torch.mm(a_norm, b_norm.transpose(0, 1)) #TODO: this keeps allocating GPU memory +RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x153600 and 2653696x1) + + +2023-05-29 17:15:31.249976 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 39, in search + query_embeddings = self.model.encode_queries( + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 148, in encode_queries + return self.model.encode(queries, batch_size=batch_size, **kwargs) + File "src/training/mteb_eval.py", line 40, in encode + all_embeddings.extend(embeddings.detach()) +NameError: name 'embeddings' is not defined + + +2023-05-29 17:20:08.362515 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 24, in cos_sim + return torch.mm(a_norm, b_norm.transpose(0, 1)) #TODO: this keeps allocating GPU memory +RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x153600 and 2653696x1) + + +2023-05-29 17:28:59.689506 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 11, in cos_sim + a = torch.tensor(a) +RuntimeError: Could not infer dtype of NoneType + + +2023-05-29 17:39:05.588611 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 11, in cos_sim + a = torch.tensor(a) +RuntimeError: Could not infer dtype of NoneType + + +2023-05-29 17:48:00.556928 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 39, in search + query_embeddings = self.model.encode_queries( + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 148, in encode_queries + return self.model.encode(queries, batch_size=batch_size, **kwargs) + File "src/training/mteb_eval.py", line 41, in encode + all_embeddings = np.stack(all_embeddings,0) + File "<__array_function__ internals>", line 200, in stack + File "/admin/home-jianz/.local/lib/python3.8/site-packages/numpy/core/shape_base.py", line 464, in stack + raise ValueError('all input arrays must have the same shape') +ValueError: all input arrays must have the same shape + + +2023-05-29 18:00:29.464698 >>> SciFact +Traceback (most recent call last): + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/evaluation/MTEB.py", line 235, in run + results = task.evaluate(model, split, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/mteb/abstasks/AbsTaskRetrieval.py", line 95, in evaluate + results = retriever.retrieve(corpus, queries) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/evaluation.py", line 23, in retrieve + return self.retriever.search(corpus, queries, self.top_k, self.score_function, **kwargs) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/exact_search.py", line 65, in search + cos_scores = self.score_functions[score_function](query_embeddings, sub_corpus_embeddings) + File "/admin/home-jianz/.local/lib/python3.8/site-packages/beir/retrieval/search/dense/util.py", line 11, in cos_sim + a = torch.tensor(a) +RuntimeError: Could not infer dtype of NoneType + + diff --git a/src/open_clip/model_configs/Siamese-mt5-xl.json b/src/open_clip/model_configs/Siamese-mt5-xl.json new file mode 100644 index 000000000..8066ae07b --- /dev/null +++ b/src/open_clip/model_configs/Siamese-mt5-xl.json @@ -0,0 +1,10 @@ +{ + "embed_dim": 1024, + "quick_gelu": true, + "text_cfg": { + "hf_model_name": "google/mt5-xl", + "hf_tokenizer_name": "google/mt5-xl", + "proj": "mlp", + "pooler_type": "mean_pooler" + } +} \ No newline at end of file diff --git a/src/training/mteb_eval.py b/src/training/mteb_eval.py index 241713bc4..f4ca1028b 100644 --- a/src/training/mteb_eval.py +++ b/src/training/mteb_eval.py @@ -1,4 +1,5 @@ import sys +import numpy as np import torch import torch.nn as nn from mteb import MTEB @@ -17,7 +18,7 @@ def __init__(self,args): self.device = 'cuda' if torch.cuda.is_available() else 'cpu' self.model.to(self.device) - def encode(self, sentences, batch_size=32, show_progress_bar=True, **kwargs): + def encode(self, sentences, batch_size=128, show_progress_bar=True, **kwargs): """ Returns a list of embeddings for the given sentences. Args: sentences (`List[str]`): List of sentences to encode @@ -34,10 +35,11 @@ def encode(self, sentences, batch_size=32, show_progress_bar=True, **kwargs): features = self.tokenizer(sentences_batch) features = features.to(self.device) with torch.no_grad(): - text_features = self.model.encode_text(features) - embeddings = [f for f in text_features.detach().cpu().numpy()] + embeddings = self.model.encode_text(features) + + all_embeddings.append(embeddings.detach()) - all_embeddings.extend(embeddings) + all_embeddings = torch.cat(all_embeddings,0) return all_embeddings @@ -47,7 +49,7 @@ def encode(self, sentences, batch_size=32, show_progress_bar=True, **kwargs): def run_mteb(args): model = CLANP_For_MTEB(args) - evaluation = MTEB(tasks=['STS22']) #task_types=['Clustering','retrieval'] + evaluation = MTEB(tasks=['MSMARCO']) #task_types=['Clustering','retrieval'] results = evaluation.run(model, eval_splits=["test"], output_folder=args.mteb_output) @@ -58,5 +60,6 @@ def run_mteb(args): # /fsx/home-jianz/logs/2023_05_11-16_04_50-model_Siamese-pythia-410m-lr_0.0001-b_1024-j_6-p_amp_bfloat16/checkpoints/epoch_1.pt # srun --account=laion --partition=g40 nodes=1 --ntasks-per-node=1 --cpus-per-task=4 --job-name=MTEB --pty bash -i - # python3 src/training/mteb_eval.py --model="Siamese-pythia-410m" --pretrained /fsx/home-jianz/logs/2023_05_11-16_04_50-model_Siamese-pythia-410m-lr_0.0001-b_1024-j_6-p_amp_bfloat16/checkpoints/epoch_1.pt --mteb-output /fsx/home-jianz/logs/mteb --context-length=77 + # srun --account laion --partition=g40 --gpus=1 --cpus-per-gpu=4 --job-name=pty --pty bash -i + # python3 src/training/mteb_eval.py --model="Siamese-xlm-roberta-large" --pretrained /fsx/home-jianz/logs/2023_05_25-09_21_35-model_Siamese-xlm-roberta-large-lr_1e-05-b_256-j_10-p_amp_bfloat16/checkpoints/epoch_latest.pt --mteb-output /fsx/home-jianz/logs/mteb --context-length=128 \ No newline at end of file From 0fa9e221520d4db0257893447d0c941077ab29ab Mon Sep 17 00:00:00 2001 From: lingjzhu Date: Tue, 20 Jun 2023 05:28:20 +0000 Subject: [PATCH 36/36] add script example --- docs/script_examples/train_clanp.sh | 70 +++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/docs/script_examples/train_clanp.sh b/docs/script_examples/train_clanp.sh index f8d1cdcf4..9feb0f0ce 100644 --- a/docs/script_examples/train_clanp.sh +++ b/docs/script_examples/train_clanp.sh @@ -1,28 +1,62 @@ -torchrun --standalone --nnodes=1 src/training/main.py \ +#!/bin/bash +#SBATCH --partition=g40 +#SBATCH --account=laion +#SBATCH --job-name=CLANP +#SBATCH --nodes=4 +#SBATCH --gpus-per-node=8 +#SBATCH --cpus-per-gpu=10 +#SBATCH --output=logs/%x_%j.out +#SBATCH --comment=laion +#SBATCH --open-mode=append +#SBATCH --exclusive +source /etc/profile.d/modules.sh +module load openmpi +module load cuda/11.7 + +export PYTHONFAULTHANDLER=1 +export CUDA_LAUNCH_BLOCKING=0 +export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"` +export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1) +export MASTER_PORT=12802 +export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l` + +echo go $COUNT_NODE +echo $HOSTNAMES + + +cd /admin/home-jianz/open_clip/src +export PYTHONPATH="$PYTHONPATH:/admin/home-jianz/open_clip/src" + +EXP_NAME="" + + + +srun --comment laion --cpu_bind=v --accel-bind=gn python3 -m training.main \ --save-frequency 1 \ --dataset-type="webdataset" \ --text-a-key="txt" \ --text-b-key="txt" \ --report-to wandb \ - --wandb-project-name="TextTextCLIP" \ - --train-data="../text.tar" \ - --train-num-samples 13560 \ - --warmup 2000 \ - --batch-size=4 \ - --precision amp_bfloat16 \ - --lr=0.001 \ - --wd=0.2 \ - --epochs=9 \ - --workers=1 \ - --model "pythia-70m-pythia-70m" \ - --seed 0 \ - --log-every-n-steps 5 \ - --model-type "CLANP" \ - --sts-val-data "lingjzhu/sts17-crosslingual" \ + --wandb-project-name="CLANP" \ + --train-data="pipe:aws s3 --cli-connect-timeout 1000 --cli-read-timeout 1000 cp s3://s-laion/pile/pile_wds/pile-{00000..01159}.tar -" \ + --train-num-samples=1000000 \ + --warmup 5000 \ + --batch-size=256 \ + --lr=0.00001 \ + --wd=0.01 \ + --epochs=100 \ + --workers=10 \ + --model="Siamese-xlm-roberta-large" \ + --save-most-recent \ + --seed 42 \ --local-loss \ --gather-with-grad \ --ddp-static-graph \ --grad-checkpointing \ - --debug \ - --unsupervised-pretraining + --precision amp_bfloat16 \ + --model-type "SiameseCLANP" \ + --unsupervised-pretraining \ + --context-length=256 \ + --logs /fsx/home-jianz/logs \ + --resume /fsx/home-jianz/logs/2023_05_25-09_21_35-model_Siamese-xlm-roberta-large-lr_1e-05-b_256-j_10-p_amp_bfloat16/checkpoints/epoch_latest.pt