Tigran Tokmajyan
commited on
Commit
·
2e668a6
1
Parent(s):
093705f
Various changes
Browse files- handler.py +11 -2
handler.py
CHANGED
@@ -11,11 +11,14 @@ formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
|
11 |
handler.setFormatter(formatter)
|
12 |
logger.addHandler(handler)
|
13 |
|
|
|
14 |
# https://www.naveedafzal.com/posts/scraping-websites-by-asking-questions-with-markuplm/
|
15 |
class EndpointHandler:
|
16 |
def __init__(self, path=""):
|
17 |
# Load model, tokenizer, and feature extractor
|
18 |
-
logger.debug("Loading model from: " + path)
|
|
|
|
|
19 |
self.processor = MarkupLMProcessor.from_pretrained("microsoft/markuplm-large-finetuned-websrc")
|
20 |
self.model = MarkupLMForQuestionAnswering.from_pretrained("microsoft/markuplm-large-finetuned-websrc")
|
21 |
|
@@ -47,6 +50,12 @@ class EndpointHandler:
|
|
47 |
predict_answer_tokens = encoding.input_ids[0, answer_start_index : answer_end_index + 1]
|
48 |
answer = self.processor.decode(predict_answer_tokens, skip_special_tokens=True)
|
49 |
|
|
|
|
|
|
|
|
|
|
|
50 |
print(f"Answer: {answer}")
|
|
|
51 |
|
52 |
-
return {"answer": answer}
|
|
|
11 |
handler.setFormatter(formatter)
|
12 |
logger.addHandler(handler)
|
13 |
|
14 |
+
# This is used https://github.com/NielsRogge/Transformers-Tutorials/blob/master/MarkupLM/Inference_with_MarkupLM_for_question_answering_on_web_pages.ipynb
|
15 |
# https://www.naveedafzal.com/posts/scraping-websites-by-asking-questions-with-markuplm/
|
16 |
class EndpointHandler:
|
17 |
def __init__(self, path=""):
|
18 |
# Load model, tokenizer, and feature extractor
|
19 |
+
# logger.debug("Loading model from: " + path)
|
20 |
+
|
21 |
+
# WE ARE CURRENTLY NOT USING OUR REPO'S MODEL
|
22 |
self.processor = MarkupLMProcessor.from_pretrained("microsoft/markuplm-large-finetuned-websrc")
|
23 |
self.model = MarkupLMForQuestionAnswering.from_pretrained("microsoft/markuplm-large-finetuned-websrc")
|
24 |
|
|
|
50 |
predict_answer_tokens = encoding.input_ids[0, answer_start_index : answer_end_index + 1]
|
51 |
answer = self.processor.decode(predict_answer_tokens, skip_special_tokens=True)
|
52 |
|
53 |
+
# Get the score
|
54 |
+
start_score = outputs.start_logits[0, answer_start_index].item()
|
55 |
+
end_score = outputs.end_logits[0, answer_end_index].item()
|
56 |
+
score = (start_score + end_score) / 2
|
57 |
+
|
58 |
print(f"Answer: {answer}")
|
59 |
+
print(f"Score: {score}")
|
60 |
|
61 |
+
return {"answer": answer, "score": score}
|