Skip to content

[wwb]: load transformers model first, then only trust_remote_code #2270

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 28, 2025
+81 −43
Merged

[wwb]: load transformers model first, then only trust_remote_code #2270

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8fc6d5d
[wwb]: load transformers model first, then only trust_remote_code
eaidova May 26, 2025
fd90804
align remote code everywhere
eaidova May 28, 2025
050f30e
Update tools/who_what_benchmark/examples/openvino_batched_eval.py
eaidova May 28, 2025
b142191
Merge branch 'master' into ea/wwb_trust_remote_code_prior
eaidova May 28, 2025
cfc64d8
align remote code everywhere
eaidova May 28, 2025
8ac4322
fix conflict
eaidova May 28, 2025
177ac04
Update tools/who_what_benchmark/whowhatbench/wwb.py
eaidova May 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions tools/who_what_benchmark/examples/openvino_batched_eval.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,14 @@

from openvino_tokenizers import convert_tokenizer
from openvino import serialize
from transformers import AutoTokenizer
from transformers import AutoTokenizer, AutoConfig

model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
MAX_NEW_TOKENS = 128
SEQS_PER_REQUEST = 5
MAX_SEQUENCES = 100


model = OVModelForCausalLM.from_pretrained(
model_id, export=True, trust_remote_code=True
)
model = OVModelForCausalLM.from_pretrained(model_id, export=True)
tokenizer = AutoTokenizer.from_pretrained(model_id)
model_path = PosixPath(tempfile.gettempdir()) / model_id
model.save_pretrained(model_path)
Expand Down
69 changes: 42 additions & 27 deletions tools/who_what_benchmark/whowhatbench/model_loaders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,16 @@ def __init__(self, model, model_dir, model_type):

if model_type == "text" or model_type == "visual-text":
try:
self.config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
except Exception:
self.config = AutoConfig.from_pretrained(model_dir)
except Exception:
self.config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
elif model_type == "text-to-image":
from diffusers import DiffusionPipeline
self.config = DiffusionPipeline.load_config(
model_dir, trust_remote_code=True)
try:
self.config = DiffusionPipeline.load_config(model_dir)
except Exception:
self.config = DiffusionPipeline.load_config(model_dir, trust_remote_code=True)


def __getattr__(self, attr):
if attr in self.__dict__:
Expand Down Expand Up @@ -89,7 +92,12 @@ def load_text_hf_pipeline(model_id, device):
model_kwargs = {}

if not torch.cuda.is_available or device.lower() == "cpu":
config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
trust_remote_code = False
try:
config = AutoConfig.from_pretrained(model_id)
except Exception:
config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
trust_remote_code = True
is_gptq = False
is_awq = False
if getattr(config, "quantization_config", None):
Expand All @@ -99,13 +107,19 @@ def load_text_hf_pipeline(model_id, device):
# infer in FP32
model_kwargs["torch_dtype"] = torch.float32
with mock_torch_cuda_is_available(is_gptq or is_awq):
model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=True, device_map="cpu", **model_kwargs)
model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=trust_remote_code, device_map="cpu", **model_kwargs)
if is_awq:
model.is_awq = is_awq
else:
model = AutoModelForCausalLM.from_pretrained(
model_id, trust_remote_code=True, device_map=device.lower(), **model_kwargs
)
try:
model = AutoModelForCausalLM.from_pretrained(
model_id, trust_remote_code=False, device_map=device.lower(), **model_kwargs
)
except Exception:
model = AutoModelForCausalLM.from_pretrained(
model_id, trust_remote_code=True, device_map=device.lower(), **model_kwargs
)

model.eval()
return model

Expand All @@ -126,7 +140,7 @@ def load_text_model(
from optimum.intel.openvino import OVModelForCausalLM
try:
model = OVModelForCausalLM.from_pretrained(
model_id, trust_remote_code=True, device=device, ov_config=ov_config
model_id, device=device, ov_config=ov_config
)
except Exception:
try:
Expand Down Expand Up @@ -177,23 +191,22 @@ def load_text2image_model(
elif use_hf:
from diffusers import DiffusionPipeline
logger.info("Using HF Transformers API")
model = DiffusionPipeline.from_pretrained(
model_id, trust_remote_code=True)
try:
model = DiffusionPipeline.from_pretrained(model_id)
except Exception:
model = DiffusionPipeline.from_pretrained(model_id, trust_remote_code=True)
else:
logger.info("Using Optimum API")
from optimum.intel import OVPipelineForText2Image
TEXT2IMAGEPipeline = OVPipelineForText2Image

try:
model = TEXT2IMAGEPipeline.from_pretrained(
model_id, trust_remote_code=True, device=device, ov_config=ov_config, safety_checker=None,
model_id, device=device, ov_config=ov_config, safety_checker=None,
)
except ValueError:
config = AutoConfig.from_pretrained(
model_id, trust_remote_code=True)
model = TEXT2IMAGEPipeline.from_pretrained(
model_id,
config=config,
trust_remote_code=True,
use_cache=True,
device=device,
Expand Down Expand Up @@ -223,19 +236,25 @@ def load_visual_text_model(
):
if use_hf:
logger.info("Using HF Transformers API")
config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
trust_remote_code = False
try:
config = AutoConfig.from_pretrained(model_id, trust_remote_code=False)
except Exception:
config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
trust_remote_code = True

try:
model = AutoModelForVision2Seq.from_pretrained(
model_id, trust_remote_code=True, device_map=device.lower()
model_id, trust_remote_code=trust_remote_code, device_map=device.lower()
)
except ValueError:
try:
model = AutoModel.from_pretrained(
model_id, trust_remote_code=True, device_map=device.lower()
model_id, trust_remote_code=trust_remote_code, device_map=device.lower()
)
except ValueError:
model = AutoModelForCausalLM.from_pretrained(
model_id, trust_remote_code=True, device_map=device.lower(), _attn_implementation="eager", use_flash_attention_2=False
model_id, trust_remote_code=trust_remote_code, device_map=device.lower(), _attn_implementation="eager", use_flash_attention_2=False
)
model.eval()
try:
Expand All @@ -255,7 +274,7 @@ def load_visual_text_model(
from optimum.intel.openvino import OVModelForVisualCausalLM
try:
model = OVModelForVisualCausalLM.from_pretrained(
model_id, trust_remote_code=True, device=device, ov_config=ov_config
model_id, device=device, ov_config=ov_config
)
except ValueError:
config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
Expand Down Expand Up @@ -301,13 +320,11 @@ def load_imagetext2image_model(
from optimum.intel.openvino import OVPipelineForImage2Image
try:
model = OVPipelineForImage2Image.from_pretrained(
model_id, trust_remote_code=True, device=device, ov_config=ov_config, safety_checker=None,
model_id, device=device, ov_config=ov_config, safety_checker=None,
)
except ValueError:
config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@andreyanufr I do not know if this code branch can be reached at all for diffusers models, but loading config in such way does not work for diffusion models:

>>> from transformers import AutoConfig
>>> config = AutoConfig.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5", trust_remote_code=True)

transformers/models/auto/configuration_auto.py", line 1151, in from_pretrained
    raise ValueError(
ValueError: Unrecognized model in stable-diffusion-v1-5/stable-diffusion-v1-5. Should have a `model_type` key in its config.json

That is why I removed this line.

model = OVPipelineForImage2Image.from_pretrained(
model_id,
config=config,
trust_remote_code=True,
use_cache=True,
device=device,
Expand Down Expand Up @@ -348,14 +365,12 @@ def load_inpainting_model(
from optimum.intel.openvino import OVPipelineForInpainting
try:
model = OVPipelineForInpainting.from_pretrained(
model_id, trust_remote_code=True, device=device, ov_config=ov_config, safety_checker=None,
model_id, device=device, ov_config=ov_config, safety_checker=None,
)
except ValueError as e:
logger.error("Failed to load inpaiting pipeline. Details:\n", e)
config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
model = OVPipelineForInpainting.from_pretrained(
model_id,
config=config,
trust_remote_code=True,
use_cache=True,
device=device,
Expand Down
10 changes: 8 additions & 2 deletions tools/who_what_benchmark/whowhatbench/whowhat_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,18 @@ def evaluate_divergency(tokenizer, data_gold, data_prediction):

class TextSimilarity:
def __init__(self, model_id) -> None:
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
trust_remote_code = False
try:
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=False)
except Exception:
trust_remote_code = True
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)

if hasattr(tokenizer, "pad_token") and tokenizer.pad_token:
pad_token = tokenizer.pad_token
else:
pad_token = tokenizer.eos_token
self.model = SentenceTransformer(model_id, tokenizer_kwargs={"pad_token": pad_token}, trust_remote_code=True)
self.model = SentenceTransformer(model_id, tokenizer_kwargs={"pad_token": pad_token}, trust_remote_code=trust_remote_code)

def evaluate(self, gt, prediction):
return evaluate_similarity(self.model, gt, prediction)
Expand Down
43 changes: 33 additions & 10 deletions tools/who_what_benchmark/whowhatbench/wwb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,17 +212,33 @@ def load_tokenizer(args):
from llama_cpp.llama_tokenizer import LlamaHFTokenizer
tokenizer = LlamaHFTokenizer.from_pretrained(args.tokenizer)
else:
try:
tokenizer = AutoTokenizer.from_pretrained(
args.tokenizer, trust_remote_code=False
)
except Exception:
tokenizer = AutoTokenizer.from_pretrained(
args.tokenizer, trust_remote_code=True
)
elif args.base_model is not None:
try:
tokenizer = AutoTokenizer.from_pretrained(
args.tokenizer, trust_remote_code=True
args.base_model, trust_remote_code=False
)
except Exception:
tokenizer = AutoTokenizer.from_pretrained(
args.base_model, trust_remote_code=True
)
elif args.base_model is not None:
tokenizer = AutoTokenizer.from_pretrained(
args.base_model, trust_remote_code=True
)
elif args.target_model is not None:
tokenizer = AutoTokenizer.from_pretrained(
args.target_model, trust_remote_code=True
)
try:
tokenizer = AutoTokenizer.from_pretrained(
args.target_model, trust_remote_code=False
)
except:
tokenizer = AutoTokenizer.from_pretrained(
args.target_model, trust_remoter_code=True
)


return tokenizer

Expand All @@ -232,13 +248,20 @@ def load_processor(args):
if model_id is None:
return None, None

config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
trust_remote_code = False
try:
config = AutoConfig.from_pretrained(model_id, trust_remote_code=False)
except Exception:
config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
trust_remote_code = True
if "llava-qwen" in config.model_type:
preprocessor_id = config.mm_vision_tower
else:
preprocessor_id = model_id

return AutoProcessor.from_pretrained(preprocessor_id, trust_remote_code=True), config
preprocessor = AutoProcessor.from_pretrained(preprocessor_id, trust_remote_code=trust_remote_code)

return preprocessor, config


def diff_strings(a: str, b: str, *, use_loguru_colors: bool = False) -> str:
Expand Down