OSINT_Tool / app.py
Canstralian's picture
Update app.py
0920957 verified
raw
history blame
4.39 kB
import streamlit as st
import requests
from bs4 import BeautifulSoup
import pandas as pd
from transformers import pipeline, AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments
from datasets import load_dataset, Dataset
# OSINT functions
def get_github_stars_forks(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}"
response = requests.get(url)
data = response.json()
return data['stargazers_count'], data['forks_count']
def get_github_issues(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/issues"
response = requests.get(url)
issues = response.json()
return len(issues)
def get_github_pull_requests(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/pulls"
response = requests.get(url)
pulls = response.json()
return len(pulls)
def get_github_license(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/license"
response = requests.get(url)
data = response.json()
return data['license']['name']
def get_last_commit(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/commits"
response = requests.get(url)
commits = response.json()
return commits[0]['commit']['committer']['date']
def get_github_workflow_status(owner, repo):
url = f"https://api.github.com/repos/{owner}/{repo}/actions/runs"
response = requests.get(url)
runs = response.json()
return runs['workflow_runs'][0]['status'] if runs['workflow_runs'] else "No workflows found"
# Function to fetch page title from a URL
def fetch_page_title(url):
try:
response = requests.get(url)
st.write(f"Fetching URL: {url} - Status Code: {response.status_code}")
if response.status_code == 200:
soup = BeautifulSoup(response.text, 'html.parser')
title = soup.title.string if soup.title else 'No title found'
return title
else:
return f"Error: Received status code {response.status_code}"
except Exception as e:
return f"An error occurred: {e}"
# Main Streamlit app
def main():
st.title("OSINT Tool")
st.write("### GitHub Repository OSINT Analysis")
st.write("Enter the GitHub repository owner and name:")
owner = st.text_input("Repository Owner")
repo = st.text_input("Repository Name")
if owner and repo:
stars, forks = get_github_stars_forks(owner, repo)
open_issues = get_github_issues(owner, repo)
open_pulls = get_github_pull_requests(owner, repo)
license_type = get_github_license(owner, repo)
last_commit = get_last_commit(owner, repo)
workflow_status = get_github_workflow_status(owner, repo)
st.write(f"Stars: {stars}, Forks: {forks}")
st.write(f"Open Issues: {open_issues}, Open Pull Requests: {open_pulls}")
st.write(f"License: {license_type}")
st.write(f"Last Commit: {last_commit}")
st.write(f"Workflow Status: {workflow_status}")
st.write("### URL Title Fetcher")
url = st.text_input("Enter a URL to fetch its title:")
if url:
title = fetch_page_title(url)
st.write(f"Title: {title}")
st.write("### Dataset Upload & Model Fine-Tuning")
dataset_file = st.file_uploader("Upload a CSV file for fine-tuning", type=["csv"])
if dataset_file:
df = pd.read_csv(dataset_file)
st.dataframe(df.head())
st.write("Select a model for fine-tuning:")
model_name = st.selectbox("Model", ["bert-base-uncased", "distilbert-base-uncased"])
if st.button("Fine-tune Model"):
if dataset_file:
dataset = Dataset.from_pandas(df)
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name)
def tokenize_function(examples):
return tokenizer(examples['text'], padding="max_length", truncation=True)
tokenized_datasets = dataset.map(tokenize_function, batched=True)
training_args = TrainingArguments(output_dir="./results", num_train_epochs=1, per_device_train_batch_size=8)
trainer = Trainer(model=model, args=training_args, train_dataset=tokenized_datasets)
trainer.train()
st.write("Model fine-tuned successfully!")
if __name__ == "__main__":
main()