chore: add more baselines (ModernBERT, CodeSage) and CodeNet fixes

downstream/Clone-detection-CodeNet/code/model.py

@@ -25,7 +25,7 @@ def forward(self, input_ids=None, p_input_ids=None, n_input_ids=None, labels=Non
         attention_mask = input_ids.ne(self.tokenizer.pad_token_id)
 
         encoder_outputs = self.encoder(input_ids, attention_mask=attention_mask)
-        if hasattr(self.args, "model_type") and self.args.model_type == "modernbert":
+        if hasattr(self.args, "model_type") and self.args.model_type in ("modernbert", "codesage"):
             outputs = self._pool(encoder_outputs[0], attention_mask)
         elif len(encoder_outputs) > 1:
             outputs = encoder_outputs[1]

downstream/Clone-detection-CodeNet/code/run.py

@@ -22,6 +22,12 @@
 
 from __future__ import absolute_import, division, print_function
 
+# Compat: Conv1D moved from modeling_utils to pytorch_utils in transformers >=4.45
+import transformers.modeling_utils as _mu
+if not hasattr(_mu, "Conv1D"):
+    from transformers.pytorch_utils import Conv1D as _Conv1D
+    _mu.Conv1D = _Conv1D
+
 import argparse
 import json
 import logging
@@ -81,8 +87,12 @@
     "roberta": (RobertaConfig, RobertaModel, RobertaTokenizer),
     "distilbert": (DistilBertConfig, DistilBertModel, DistilBertTokenizer),
     "modernbert": (AutoConfig, AutoModel, AutoTokenizer),
+    "codesage": (AutoConfig, AutoModel, AutoTokenizer),
 }
 
+# Model types that use custom code hosted on HuggingFace Hub.
+_TRUST_REMOTE_CODE_TYPES = {"codesage"}
+
 
 class InputFeatures(object):
     """A single training/test features for a example."""
@@ -826,16 +836,25 @@ def main():
         )
 
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    trust_remote = args.model_type in _TRUST_REMOTE_CODE_TYPES
     config = config_class.from_pretrained(
         args.config_name if args.config_name else args.model_name_or_path,
         cache_dir=args.cache_dir if args.cache_dir else None,
+        trust_remote_code=trust_remote,
     )
     config.num_labels = 1
     tokenizer = tokenizer_class.from_pretrained(
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,
         cache_dir=args.cache_dir if args.cache_dir else None,
-    )
+        trust_remote_code=trust_remote,
+    )
+    if tokenizer.cls_token is None:
+        tokenizer.cls_token = tokenizer.eos_token
+    if tokenizer.sep_token is None:
+        tokenizer.sep_token = tokenizer.eos_token
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
     if args.block_size <= 0:
         args.block_size = (
             tokenizer.max_len_single_sentence
@@ -847,6 +866,7 @@ def main():
             from_tf=bool(".ckpt" in args.model_name_or_path),
             config=config,
             cache_dir=args.cache_dir if args.cache_dir else None,
+            trust_remote_code=trust_remote,
         )
     else:
         model = model_class(config)

downstream/Clone-detection-CodeNet/dataset/download.py

@@ -16,7 +16,7 @@ def get_tar_name(subset: str) -> str:
 
 def get_url(subset: str) -> str:
     tar_name = get_tar_name(subset)
-    url = f"https://dax-cdn.cdn.appdomain.cloud/dax-project-codenet/1.0.0/{tar_name}"
+    url = f"https://codait-cos-dax.s3.us.cloud-object-storage.appdomain.cloud/dax-project-codenet/1.0.0/{tar_name}"
 
     return url
 

downstream/Clone-detection-CodeNet/dataset/get_codenet.py

@@ -21,7 +21,7 @@ def process(subset: str):
             case "Python800":
                 augment_Python800(
                     input_file_path=test_file_path,
-                    ouput_file_path=aug_file_path,
+                    output_file_path=aug_file_path,
                 )
             case "Java250":
                 augment_Java250(

downstream/Clone-detection-POJ104/code/model.py

@@ -24,7 +24,7 @@ def forward(self, input_ids=None, p_input_ids=None, n_input_ids=None, labels=Non
         attention_mask = input_ids.ne(self.tokenizer.pad_token_id)
 
         encoder_outputs = self.encoder(input_ids, attention_mask=attention_mask)
-        if hasattr(self.args, "model_type") and self.args.model_type == "modernbert":
+        if hasattr(self.args, "model_type") and self.args.model_type in ("modernbert", "codesage"):
             outputs = self._pool(encoder_outputs[0], attention_mask)
         elif len(encoder_outputs) > 1:
             outputs = encoder_outputs[1]

downstream/Clone-detection-POJ104/code/run.py

@@ -22,6 +22,12 @@
 
 from __future__ import absolute_import, division, print_function
 
+# Compat: Conv1D moved from modeling_utils to pytorch_utils in transformers >=4.45
+import transformers.modeling_utils as _mu
+if not hasattr(_mu, "Conv1D"):
+    from transformers.pytorch_utils import Conv1D as _Conv1D
+    _mu.Conv1D = _Conv1D
+
 import argparse
 import json
 import logging
@@ -81,8 +87,12 @@
     "roberta": (RobertaConfig, RobertaModel, RobertaTokenizer),
     "distilbert": (DistilBertConfig, DistilBertModel, DistilBertTokenizer),
     "modernbert": (AutoConfig, AutoModel, AutoTokenizer),
+    "codesage": (AutoConfig, AutoModel, AutoTokenizer),
 }
 
+# Model types that use custom code hosted on HuggingFace Hub.
+_TRUST_REMOTE_CODE_TYPES = {"codesage"}
+
 
 class InputFeatures(object):
     """A single training/test features for a example."""
@@ -826,16 +836,25 @@ def main():
         )
 
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    trust_remote = args.model_type in _TRUST_REMOTE_CODE_TYPES
     config = config_class.from_pretrained(
         args.config_name if args.config_name else args.model_name_or_path,
         cache_dir=args.cache_dir if args.cache_dir else None,
+        trust_remote_code=trust_remote,
     )
     config.num_labels = 1
     tokenizer = tokenizer_class.from_pretrained(
         args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
         do_lower_case=args.do_lower_case,
         cache_dir=args.cache_dir if args.cache_dir else None,
-    )
+        trust_remote_code=trust_remote,
+    )
+    if tokenizer.cls_token is None:
+        tokenizer.cls_token = tokenizer.eos_token
+    if tokenizer.sep_token is None:
+        tokenizer.sep_token = tokenizer.eos_token
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
     if args.block_size <= 0:
         args.block_size = (
             tokenizer.max_len_single_sentence
@@ -847,6 +866,7 @@ def main():
             from_tf=bool(".ckpt" in args.model_name_or_path),
             config=config,
             cache_dir=args.cache_dir if args.cache_dir else None,
+            trust_remote_code=trust_remote,
         )
     else:
         model = model_class(config)

downstream/Code-classification-CodeNet/code/run.py

@@ -21,6 +21,12 @@
 
 from __future__ import absolute_import, division, print_function
 
+# Compat: Conv1D moved from modeling_utils to pytorch_utils in transformers >=4.45
+import transformers.modeling_utils as _mu
+if not hasattr(_mu, "Conv1D"):
+    from transformers.pytorch_utils import Conv1D as _Conv1D
+    _mu.Conv1D = _Conv1D
+
 import argparse
 import json
 import logging
@@ -51,8 +57,12 @@
 MODEL_CLASSES = {
     "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
     "modernbert": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer),
+    "codesage": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer),
 }
 
+# Model types that use custom code hosted on HuggingFace Hub.
+_TRUST_REMOTE_CODE_TYPES = {"codesage"}
+
 logger = logging.getLogger(__name__)
 
 
@@ -430,12 +440,24 @@ def main():
     set_seed(args.seed)
 
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(args.model_name_or_path)
+    trust_remote = args.model_type in _TRUST_REMOTE_CODE_TYPES
+    config = config_class.from_pretrained(
+        args.model_name_or_path, trust_remote_code=trust_remote
+    )
     config.num_labels = 104
     tokenizer = tokenizer_class.from_pretrained(
-        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path
+        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+        trust_remote_code=trust_remote,
+    )
+    if tokenizer.cls_token is None:
+        tokenizer.cls_token = tokenizer.eos_token
+    if tokenizer.sep_token is None:
+        tokenizer.sep_token = tokenizer.eos_token
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+    model = model_class.from_pretrained(
+        args.model_name_or_path, config=config, trust_remote_code=trust_remote
     )
-    model = model_class.from_pretrained(args.model_name_or_path, config=config)
 
     model = Model(model, config, tokenizer, args)
 

downstream/Code-classification-CodeNet/dataset/augment_Python800.py

@@ -24,7 +24,7 @@ def validate_jsonl_path(path: str):
 
 def main(
     input_file_path: str,
-    ouput_file_path: str,
+    output_file_path: str,
     nproc: int = cpu_count(),
 ):
     validate_jsonl_path(input_file_path)
@@ -36,11 +36,11 @@ def main(
         augmented_dataset = pool.map(augment_accumulatively, all_test_json)
         augmented_jsonl = pool.map(lambda j: json.dumps(asdict(j)), augmented_dataset)
 
-    with open(ouput_file_path, "w", encoding=JSON_ENCODING) as f:
+    with open(output_file_path, "w", encoding=JSON_ENCODING) as f:
         for aj in augmented_jsonl:
             f.write(aj + "\n")
 
-    print(f"Successfully added transformed data to {ouput_file_path}")
+    print(f"Successfully added transformed data to {output_file_path}")
 
 
 if __name__ == "__main__":

downstream/Code-classification-CodeNet/dataset/download.py

@@ -16,7 +16,7 @@ def get_tar_name(subset: str) -> str:
 
 def get_url(subset: str) -> str:
     tar_name = get_tar_name(subset)
-    url = f"https://dax-cdn.cdn.appdomain.cloud/dax-project-codenet/1.0.0/{tar_name}"
+    url = f"https://codait-cos-dax.s3.us.cloud-object-storage.appdomain.cloud/dax-project-codenet/1.0.0/{tar_name}"
 
     return url
 

downstream/Code-classification-CodeNet/dataset/get_codenet.py

@@ -21,7 +21,7 @@ def process(subset: str):
             case "Python800":
                 augment_Python800(
                     input_file_path=test_file_path,
-                    ouput_file_path=aug_file_path,
+                    output_file_path=aug_file_path,
                 )
             case "Java250":
                 augment_Java250(

downstream/Code-classification-POJ104/code/run.py

@@ -21,6 +21,12 @@
 
 from __future__ import absolute_import, division, print_function
 
+# Compat: Conv1D moved from modeling_utils to pytorch_utils in transformers >=4.45
+import transformers.modeling_utils as _mu
+if not hasattr(_mu, "Conv1D"):
+    from transformers.pytorch_utils import Conv1D as _Conv1D
+    _mu.Conv1D = _Conv1D
+
 import argparse
 import json
 import logging
@@ -51,15 +57,24 @@
 MODEL_CLASSES = {
     "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
     "modernbert": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer),
+    "codesage": (AutoConfig, AutoModelForSequenceClassification, AutoTokenizer),
 }
 
+# Model types that use custom code hosted on HuggingFace Hub.
+_TRUST_REMOTE_CODE_TYPES = {"codesage"}
+
 
 def resolve_model_type(model_type, model_name_or_path):
     if model_type in MODEL_CLASSES:
         return model_type
     candidate = model_name_or_path or model_type or ""
     name = candidate.lower()
-    resolved = "modernbert" if "modernbert" in name else "roberta"
+    if "modernbert" in name:
+        resolved = "modernbert"
+    elif "codesage" in name:
+        resolved = "codesage"
+    else:
+        resolved = "roberta"
     logger.warning(
         "Unknown model_type '%s'; inferring '%s' from '%s'.",
         model_type,
@@ -448,12 +463,24 @@ def main():
 
     args.model_type = resolve_model_type(args.model_type, args.model_name_or_path)
     config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
-    config = config_class.from_pretrained(args.model_name_or_path)
+    trust_remote = args.model_type in _TRUST_REMOTE_CODE_TYPES
+    config = config_class.from_pretrained(
+        args.model_name_or_path, trust_remote_code=trust_remote
+    )
     config.num_labels = 104
     tokenizer = tokenizer_class.from_pretrained(
-        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path
+        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+        trust_remote_code=trust_remote,
+    )
+    if tokenizer.cls_token is None:
+        tokenizer.cls_token = tokenizer.eos_token
+    if tokenizer.sep_token is None:
+        tokenizer.sep_token = tokenizer.eos_token
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+    model = model_class.from_pretrained(
+        args.model_name_or_path, config=config, trust_remote_code=trust_remote
     )
-    model = model_class.from_pretrained(args.model_name_or_path, config=config)
 
     model = Model(model, config, tokenizer, args)
 

experiments_downstream/Clone-detection-BigCloneBench/clone-bcb_modernbert.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Downstream evaluation: Clone-detection-BigCloneBench with modernbert
+set -euo pipefail
+
+# Parse CUDA device argument (default: 0)
+CUDA_DEVICE="${1:-0}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE"
+
+cd "$ROOT_DIR/downstream/Clone-detection-BigCloneBench"
+./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Clone-detection-BigCloneBench"

experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_codesage.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Downstream evaluation: Clone-detection-CodeNet (C++1400) with codesage
+set -euo pipefail
+
+# Parse CUDA device argument (default: 0)
+CUDA_DEVICE="${1:-0}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE"
+
+cd "$ROOT_DIR/downstream/Clone-detection-CodeNet"
+./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Clone-detection-CodeNet" C++1400 codesage codesage/codesage-small

experiments_downstream/Clone-detection-CodeNet/clone-codenet_C++1400_modernbert.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Downstream evaluation: Clone-detection-CodeNet (C++1400) with modernbert
+set -euo pipefail
+
+# Parse CUDA device argument (default: 0)
+CUDA_DEVICE="${1:-0}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE"
+
+cd "$ROOT_DIR/downstream/Clone-detection-CodeNet"
+./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Clone-detection-CodeNet" C++1400 modernbert answerdotai/ModernBERT-base

experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_codesage.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Downstream evaluation: Clone-detection-CodeNet (Java250) with codesage
+set -euo pipefail
+
+# Parse CUDA device argument (default: 0)
+CUDA_DEVICE="${1:-0}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE"
+
+cd "$ROOT_DIR/downstream/Clone-detection-CodeNet"
+./run.sh codesage/codesage-small "$ROOT_DIR/results/codesage/Clone-detection-CodeNet" Java250 codesage codesage/codesage-small

experiments_downstream/Clone-detection-CodeNet/clone-codenet_Java250_modernbert.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Downstream evaluation: Clone-detection-CodeNet (Java250) with modernbert
+set -euo pipefail
+
+# Parse CUDA device argument (default: 0)
+CUDA_DEVICE="${1:-0}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+export CUDA_VISIBLE_DEVICES="$CUDA_DEVICE"
+
+cd "$ROOT_DIR/downstream/Clone-detection-CodeNet"
+./run.sh answerdotai/ModernBERT-base "$ROOT_DIR/results/modernbert/Clone-detection-CodeNet" Java250 modernbert answerdotai/ModernBERT-base