Upload files with `vila-upload`.

Upload auto_processor.py
Upload modeling_vila.py
Upload llm/tokenizer_config.json

auto_processor.py

@@ -153,16 +153,19 @@ class VILAProcessor(ProcessorMixin):
     # image_processor_class = "VILAImageProcessor"
     # tokenizer_class = ("VILATokenizer", "VILATokenizerFast")
 
-    def __init__(self, image_processor=None, tokenizer=None, chat_template=None, config=None, **kwargs):
+    def __init__(self, image_processor=None, tokenizer=None, chat_template=None, config=None, padding_side="left", **kwargs):
         self.image_token = MEDIA_TOKENS["image"]
         self.video_token = MEDIA_TOKENS["video"]
         self.config = config
         self.image_processor = image_processor
         self.tokenizer = tokenizer
+        self.padding_side = padding_side
+
+        # This is a special setting for Qwen. 
         # self.pad_token_id = tokenizer.pad_token_id
-        self.pad_token_id = self.tokenizer("<|endoftext|>").input_ids[0]
+        self.pad_token_id = self.tokenizer("<|endoftext|>").input_ids[0] # 151643
         self.eos_token_id = self.tokenizer.eos_token_id
-        # self.pad_token_id = 151643
+
         super().__init__(image_processor, tokenizer, chat_template=chat_template)
 
     @staticmethod
@@ -193,6 +196,7 @@ class VILAProcessor(ProcessorMixin):
     ) -> tuple[list[PIL.Image.Image] | None, list[torch.Tensor | list[PIL.Image.Image]] | None, Optional[dict]]:
         """
         referernce from qwen_vl_utils
+        NVILA does not depend on the function, but the interface is the same.
         """
         vision_infos = extract_vision_info(conversations)
         ## Read images or videos
@@ -233,12 +237,12 @@ class VILAProcessor(ProcessorMixin):
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+        padding_side = kwargs.get("padding_side", "left")
         if os.path.isdir(pretrained_model_name_or_path):
             pretrained_model_name_or_path = pretrained_model_name_or_path
         else:
             print(f"pretrained_model_name_or_path {pretrained_model_name_or_path} is not a directory, downloading")
             from huggingface_hub import snapshot_download
-
             pretrained_model_name_or_path = snapshot_download(pretrained_model_name_or_path)
 
         image_processor = AutoImageProcessor.from_pretrained(
@@ -248,7 +252,7 @@ class VILAProcessor(ProcessorMixin):
             osp.join(pretrained_model_name_or_path, "llm"), trust_remote_code=True
         )
         config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True)
-        return cls(image_processor=image_processor, tokenizer=tokenizer, config=config)
+        return cls(image_processor=image_processor, tokenizer=tokenizer, config=config, padding_side=padding_side)
 
     def __repr__(self):
         # NOTE(ligeng):  hard coded image_processor to avoid serialization error. Dirty fix
@@ -275,7 +279,7 @@ class VILAProcessor(ProcessorMixin):
         if kwargs.get("text", None) is not None:
             conversation = kwargs.get("text")
         assert conversation is not None, "`conversation` or `text` is required"
-        padding_side = kwargs.get("padding_side", "left")
+        padding_side = kwargs.get("padding_side", self.padding_side)
 
         input_ids_list = []
         attention_mask = []
@@ -289,7 +293,8 @@ class VILAProcessor(ProcessorMixin):
                 media[name] += feat.media[name]
             for name in feat.media_config:
                 media_config[name].update(feat.media_config[name])
-
+        
+        # pad the input_ids to batchfy
         input_ids = pad_fn(
             input_ids_list,
             padding_value=self.pad_token_id,
@@ -299,9 +304,10 @@ class VILAProcessor(ProcessorMixin):
         attention_mask = torch.ones_like(input_ids, dtype=torch.bool)
         attention_mask[input_ids == self.pad_token_id] = False
         # print("[DEBUGAAA]", self.pad_token_id, self.tokenizer.pad_token_id); exit(0)
-
+        input_texts = self.tokenizer.batch_decode(input_ids)
         return BatchFeature(
             data={
+                "input_texts": input_texts,
                 "input_ids": input_ids,
                 "attention_mask": attention_mask,
                 "media": media,
@@ -329,6 +335,8 @@ class VILAProcessor(ProcessorMixin):
                     self.config.image_processor = self.image_processor
                     if self.config.image_aspect_ratio == "dynamic":
                         images = process_image(media["image"][0], self.config, None, enable_dynamic_res=True).half()
+                        # print("DEBUG", len(images)); input()
+                        # NOTE: this only works for images appears at the first conversation
                         conversation[0]["value"] = conversation[0]["value"].replace(
                             DEFAULT_IMAGE_TOKEN, f"{DEFAULT_IMAGE_TOKEN}\n" * images.shape[0]
                         )
@@ -352,7 +360,7 @@ class VILAProcessor(ProcessorMixin):
                 raise ValueError(f"Unsupported media type: {name}")
 
         inputs = tokenize_conversation(conversation, self.tokenizer, add_generation_prompt=True, return_ids_only=False)
-        input_ids = inputs.input_ids[0].cuda().unsqueeze(0)
+        input_ids = inputs.input_ids[0].unsqueeze(0)#.cuda()
         attention_mask = torch.ones_like(input_ids, dtype=torch.bool)
         return BatchFeature(
             data={

llm/tokenizer_config.json

@@ -78,7 +78,7 @@
   "legacy": false,
   "model_max_length": 4096,
   "pad_token": "[PAD]",
-  "padding_side": "right",
+  "padding_side": "left",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null

modeling_vila.py

@@ -201,17 +201,19 @@ class VILAPretrainedModel(PreTrainedModel):
         else:
             raise ValueError("`llm_cfg` `mm_projector_cfg` `vision_tower_cfg` not found in the config.")
 
-        # loading on cpu by default
-        device_map = kwargs.get("device_map", "cpu")
+        # loading on auto by default
+        device_map = kwargs.get("device_map", "auto")
         self.mm_projector = build_mm_projector(mm_projector_cfg, config)
         self.vision_tower = build_vision_tower(vision_tower_cfg, config)
-        if "auto" in device_map or "cuda" in device_map:
+        if device_map in ["auto", "cuda"]:
             self.mm_projector = self.mm_projector.cuda()
             self.vision_tower = self.vision_tower.cuda()
         # set device_map auto can autoamtically shard llm to different devices
         self.llm, self.tokenizer = self.init_llm(llm_cfg, config, device_map=device_map)
 
-        # NOTE(ligeng): need to add other decoders from config
+        # NOTE(ligeng): hard code to set padding_side to left
+        self.tokenizer.padding_side = "left"
+        # TODO(ligeng): need to add other decoders from config
         self.encoders = {"image": BasicImageEncoder(self), "video": BasicVideoEncoder(self)}
 
         self.post_config()
@@ -418,6 +420,7 @@ class VILAPretrainedModel(PreTrainedModel):
         weights_only: bool = True,
         **kwargs,
     ):
+        # print("DEBUG2", kwargs); input()
         config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True)
         return cls._from_config(config, **kwargs)
 
@@ -450,6 +453,10 @@ class VILAPretrainedModel(PreTrainedModel):
         self.vision_tower = self.vision_tower.to(torch.float16)
         ######################################################################
         self.training = self.llm.training
+        if self.training:
+            self.train()
+        else:
+            self.eval()
         ## configuration
         if getattr(self.config, "llm_cfg", None) is None:
             self.config.llm_cfg = self.llm.config
@@ -595,10 +602,6 @@ class VILAForCasualLM(VILAPretrainedModel):
         return image_features
 
     def train(self, mode: bool = True):
-        if mode:
-            self.tokenizer.padding_side = "right"
-        else:
-            self.tokenizer.padding_side = "left"
         super().train(mode)
         return self
 
@@ -657,6 +660,7 @@ class VILAForCasualLM(VILAPretrainedModel):
                     input = media_embeds[name].popleft()
                     label = torch.full([input.shape[0]], IGNORE_INDEX, device=labels[k].device, dtype=labels[k].dtype)
                 elif input_ids[k][pos].item() in self.pad_token_list:
+                    # skip pad tokens
                     end = pos + 1
                     pos = end
                     continue
@@ -1102,6 +1106,12 @@ class VILAForCasualLM(VILAPretrainedModel):
         input_tokens:     36000       001 002 003 004
         input_emds:     <media emd>   001 002 003 004
         """
+        # NOTE: hard code to move to GPU
+        input_ids = input_ids.cuda()
+        media = {k: [v.cuda() for v in media[k]] for k in media}
+        if attention_mask is not None:
+            attention_mask = attention_mask.cuda()
+
         # TODO: there is still a padding left vs right issue unsovled here.
         # print("prev args:",input_ids.shape, media, media_config, None, attention_mask)
         inputs_embeds, _, attention_mask = self._embed(input_ids, media, media_config, None, attention_mask)
@@ -1110,11 +1120,11 @@ class VILAForCasualLM(VILAPretrainedModel):
         output_ids = self.llm.generate(inputs_embeds=inputs_embeds, attention_mask=attention_mask, **generation_kwargs)
         # print("output_ids", self.tokenizer.batch_decode(output_ids))
         # input("wait for debug")
+        
         if return_output_ids_only:
             return_value = output_ids
         else:
             # by default, return the input_ids and output_ids concatenated to keep consistency with the community VLMs like qwen
-            # print(f"[DEBUG REMOTE] input_ids: {input_ids.shape}, output_ids: {output_ids.shape} attention_mask: {attention_mask.shape} {generation_kwargs=}"); exit(0)
             generation_config = generation_kwargs.get("generation_config", None)
             if generation_config is not None:
                 num_generations = generation_config.num_return_sequences