feat: another simple approach

app.py

@@ -1,33 +1,27 @@
 import gradio as gr
 import spaces
-from transformers import pipeline
-import logging
-
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-
-# Load model and tokenizer using pipeline
-logging.info("Loading model and tokenizer...")
-model_name = "speakleash/Bielik-11B-v2.3-Instruct"
-pipe = pipeline("text-generation", model=model_name)
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
 
 
 @spaces.GPU
-def process_text(input_text):
-    messages = [{"role": "user", "content": input_text}]
-    outputs = pipe(messages)
-    # Process outputs as needed
-    return outputs
+def test():
+    device = torch.device("cuda")
+    tokenizer = AutoTokenizer.from_pretrained("speakleash/Bielik-7B-Instruct-v0.1")
+    model = AutoModelForCausalLM.from_pretrained(
+            "speakleash/Bielik-7B-Instruct-v0.1-GGUF",
+            model_file="bielik-7b-instruct-v0.1.Q4_K_M.gguf",
+            model_type="mistral", gpu_layers=50, hf=True).to(device)
+
+    inputs = tokenizer("Cześć Bielik, jak się masz?", return_tensors="pt").to(device)
 
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs, max_new_tokens=128, pad_token_id=tokenizer.eos_token_id
+        )
 
-def generate(text):
-    hardcoded_prompt = "Stwórz zwięzłe podsumowanie tekstu, zachowując kluczowe punkty. Maksymalnie 3 zdania"
-    combined_text = hardcoded_prompt + text
-    return process_text(combined_text)
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
 
-gr.Interface(
-    fn=generate,
-    inputs=gr.Text(),
-    outputs=gr.Text(),
-).launch()
+demo = gr.Interface(fn=test, inputs=None, outputs=gr.Text())
+demo.launch()