Hhhh

Running

File size: 17,108 Bytes

5b4adb6

from main import *
from tts_api import *
from stt_api import *
from sentiment_api import *
from imagegen_api import *
from musicgen_api import *
from translation_api import *
from codegen_api import *
from text_to_video_api import *
from summarization_api import *
from image_to_3d_api import *
from flask import Flask, request, jsonify, Response, send_file, stream_with_context
from flask_cors import CORS
import torch
import torch.nn.functional as F
import torchaudio
import numpy as np
from PIL import Image
import io
import tempfile
import queue
import json
import base64
from markupsafe import Markup
from markupsafe import escape

app = Flask(__name__)
CORS(app)

html_code = """<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>AI Conversational Avatar</title>
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/animate.css/4.1.1/animate.min.css"/>
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css"/>
    <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
    <style>
        body {
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            background: #f0f0f0;
            color: #333;
            margin: 0;
            padding: 0;
            display: flex;
            flex-direction: column;
            align-items: center;
            min-height: 100vh;
        }
        .container {
            width: 95%;
            max-width: 900px;
            padding: 20px;
            background-color: #fff;
            box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
            border-radius: 8px;
            margin-top: 20px;
            margin-bottom: 20px;
            display: flex;
            flex-direction: column;
        }
        .header {
            text-align: center;
            margin-bottom: 20px;
        }
        .header h1 {
            font-size: 2em;
            color: #333;
        }
        .form-group {
            margin-bottom: 15px;
        }
        .form-group textarea, .form-group input[type="text"] {
            width: 100%;
            padding: 10px;
            border: 1px solid #ccc;
            border-radius: 5px;
            font-size: 16px;
            box-sizing: border-box;
        }
        button, #recordButton, #stopButton {
            padding: 10px 15px;
            border: none;
            border-radius: 5px;
            background-color: #007bff;
            color: white;
            font-size: 18px;
            cursor: pointer;
            transition: background-color 0.3s ease;
            margin-right: 5px;
        }
        button:hover, #recordButton:hover, #stopButton:hover {
            background-color: #0056b3;
        }
        #output {
            margin-top: 20px;
            padding: 15px;
            border: 1px solid #ddd;
            border-radius: 5px;
            background-color: #f9f9f9;
            white-space: pre-wrap;
            word-break: break-word;
            overflow-y: auto;
            max-height: 300px;
        }
        #videoOutput {
            margin-top: 20px;
            border: 1px solid #ddd;
            border-radius: 5px;
            overflow: hidden;
        }
        #videoOutput video {
            width: 100%;
            display: block;
        }
        #animatedText {
            position: fixed;
            top: 20px;
            left: 20px;
            font-size: 1.5em;
            color: rgba(0, 0, 0, 0.1);
            pointer-events: none;
            z-index: -1;
        }
        #transcriptionOutput {
            margin-top: 10px;
            padding: 10px;
            border: 1px solid #ddd;
            border-radius: 5px;
            background-color: #f9f9f9;
            font-size: 14px;
            word-break: break-word;
        }
        @media (max-width: 768px) {
            .container {
                width: 98%;
                margin-top: 10px;
                margin-bottom: 10px;
                padding: 15px;
            }
            .header h1 {
                font-size: 1.8em;
            }
            .form-group textarea, .form-group input[type="text"] {
                font-size: 14px;
                padding: 8px;
            }
            button, #recordButton, #stopButton {
                font-size: 16px;
                padding: 8px 12px;
            }
            #output, #transcriptionOutput {
                font-size: 14px;
                padding: 10px;
                margin-top: 15px;
            }
        }
    </style>
</head>
<body>
<div id="animatedText" class="animated-text animate__animated animate__fadeIn animate__infinite infinite">AI POWERED</div>
<div class="container">
    <div class="header animate__animated animate__fadeInDown">
        <h1>Conversational Avatar</h1>
    </div>
    <div class="form-group animate__animated animate__fadeInLeft">
        <textarea id="textInput" rows="3" placeholder="Or type your request here"></textarea>
    </div>
    <div class="form-group animate__animated animate__fadeInRight" style="text-align: center;">
        <button onclick="generateResponse()" class="animate__animated animate__fadeInUp">Generate Avatar Response</button>
    </div>

    <div style="text-align: center; margin-bottom: 15px;">
        <button id="recordButton" class="animate__animated animate__fadeInUp"><i class="fas fa-microphone"></i> Start Recording</button>
        <button id="stopButton" class="animate__animated animate__fadeInUp" disabled><i class="fas fa-stop-circle"></i> Stop Recording</button>
    </div>

    <div id="transcriptionOutput" class="animate__animated animate__fadeIn">
        <strong>Transcription:</strong>
        <span id="transcriptionText"></span>
    </div>

    <div id="output" class="animate__animated animate__fadeIn">
        <strong>Response:</strong><br>
        <span id="responseText"></span>
    </div>

    <div id="videoOutput" class="animate__animated animate__fadeIn">
        <video id="avatarVideo" controls></video>
    </div>
</div>

<script>
    let mediaRecorder;
    let audioChunks = [];
    let lastResponse = "";
    let accumulatedText = "";
    let eventSource = null;
    let audioURL;

    const recordButton = document.getElementById('recordButton');
    const stopButton = document.getElementById('stopButton');
    const transcriptionTextSpan = document.getElementById('transcriptionText');
    const responseTextSpan = document.getElementById('responseText');
    const avatarVideoPlayer = document.getElementById('avatarVideo');
    const textInputField = document.getElementById('textInput');

    recordButton.onclick = async () => {
        try {
            const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
            mediaRecorder = new MediaRecorder(stream);
            audioChunks = [];

            mediaRecorder.ondataavailable = event => {
                audioChunks.push(event.data);
            };

            mediaRecorder.onstop = async () => {
                const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
                const formData = new FormData();
                formData.append('audio', audioBlob, 'recording.wav');

                transcriptionTextSpan.innerText = "Transcribing...";
                responseTextSpan.innerText = "";
                avatarVideoPlayer.src = "";

                try {
                    const sttResponse = await fetch('/api/v1/stt', {
                        method: 'POST',
                        body: formData
                    });

                    if (!sttResponse.ok) {
                        throw new Error(\`HTTP error! status: ${sttResponse.status}\`);
                    }

                    const sttData = await sttResponse.json();
                    const transcribedText = sttData.text;
                    transcriptionTextSpan.innerText = transcribedText || "Transcription failed.";

                    if (transcribedText) {
                        await generateAvatarVideoResponse(transcribedText);
                    }

                } catch (error) {
                    console.error("STT or subsequent error:", error);
                    transcriptionTextSpan.innerText = "Transcription error.";
                    responseTextSpan.innerText = "Error processing audio.";
                } finally {
                    recordButton.disabled = false;
                    stopButton.disabled = true;
                }
            };

            recordButton.disabled = true;
            stopButton.disabled = false;
            transcriptionTextSpan.innerText = "Recording...";
            mediaRecorder.start();

        } catch (error) {
            console.error("Error accessing microphone:", error);
            transcriptionTextSpan.innerText = "Microphone access denied or error.";
            recordButton.disabled = false;
            stopButton.disabled = true;
        }
    };

    stopButton.onclick = () => {
        if (mediaRecorder && mediaRecorder.state === "recording") {
            transcriptionTextSpan.innerText = "Processing...";
            mediaRecorder.stop();
            recordButton.disabled = true;
            stopButton.disabled = true;
        }
    };

    async function generateResponse() {
        const inputText = textInputField.value;
        if (!inputText.trim()) {
            alert("Please enter text or record audio.");
            return;
        }
        transcriptionTextSpan.innerText = inputText;
        await generateAvatarVideoResponse(inputText);
    }


    async function generateAvatarVideoResponse(inputText) {
        responseTextSpan.innerText = "Generating response...";
        avatarVideoPlayer.src = "";
        accumulatedText = "";
        lastResponse = "";

        const temp = 0.7;
        const top_k_val = 40;
        const top_p_val = 0.0;
        const repetition_penalty_val = 1.2;

        const requestData = {
            text: inputText,
            temp: temp,
            top_k: top_k_val,
            top_p: top_p_val,
            reppenalty: repetition_penalty_val
        };

        if (eventSource) {
            eventSource.close();
        }

        eventSource = new EventSource('/api/v1/generate_stream?' + new URLSearchParams(requestData).toString());

        eventSource.onmessage = async function(event) {
            if (event.data === "<END_STREAM>") {
                eventSource.close();
                const currentResponse = accumulatedText.replace("<|endoftext|>", "").replace(/\s+(?=[.,，。])/g, '').trim();
                if (currentResponse === lastResponse.trim()) {
                    accumulatedText = "**Response is repetitive. Please try again or rephrase your query.**";
                } else {
                    lastResponse = currentResponse;
                }
                responseTextSpan.innerHTML = marked.parse(accumulatedText);

                try {
                    const ttsResponse = await fetch('/api/v1/tts', {
                        method: 'POST',
                        headers: {
                            'Content-Type': 'application/json'
                        },
                        body: JSON.stringify({ text: currentResponse })
                    });

                    if (!ttsResponse.ok) {
                        throw new Error(\`TTS HTTP error! status: ${ttsResponse.status}\`);
                    }

                    const ttsBlob = await ttsResponse.blob();
                    audioURL = URL.createObjectURL(ttsBlob);


                    const sadTalkerResponse = await fetch('/api/v1/sadtalker', {
                        method: 'POST',
                        body: new URLSearchParams({
                            'source_image': './examples/source_image/full_body_female.png',
                            'driven_audio': audioURL,
                            'preprocess': 'full',
                            'still_mode': false,
                            'use_enhancer': true
                        })
                    });

                    if (!sadTalkerResponse.ok) {
                        throw new Error(\`SadTalker HTTP error! status: ${sadTalkerResponse.status}\`);
                    }

                    const sadTalkerData = await sadTalkerResponse.json();
                    const videoURL = sadTalkerData.video_url;
                    avatarVideoPlayer.src = videoURL;


                } catch (ttsError) {
                    console.error("TTS or SadTalker error:", ttsError);
                    responseTextSpan.innerHTML += "<br><br>Error generating audio or video avatar.";
                }

                return;
            }
            accumulatedText += event.data;
            let partialText = accumulatedText.replace("<|endoftext|>", "").replace(/\s+(?=[.,，。])/g, '').trim();
            responseTextSpan.innerHTML = marked.parse(partialText);
        };

        eventSource.onerror = function(error) {
            console.error("SSE error", error);
            eventSource.close();
            responseTextSpan.innerText = "Error generating response stream.";
        };

        const outputDiv = document.getElementById("output");
        outputDiv.classList.add("show");
    }


</script>
</body>
</html>
"""

feedback_queue = queue.Queue()


@app.route("/")
def index():
    return html_code

@app.route("/api/v1/generate_stream", methods=["GET"])
def generate_stream():
    text = request.args.get("text", "")
    temp = float(request.args.get("temp", 0.7))
    top_k = int(request.args.get("top_k", 40))
    top_p = float(request.args.get("top_p", 0.0))
    reppenalty = float(request.args.get("reppenalty", 1.2))
    response_queue = queue.Queue()
    reasoning_queue.put({
        'text_input': text,
        'temperature': temp,
        'top_k': top_k,
        'top_p': top_p,
        'repetition_penalty': reppenalty,
        'response_queue': response_queue
    })
    @stream_with_context
    def event_stream():
        while True:
            output = response_queue.get()
            if "error" in output:
                yield "data: <ERROR>\n\n"
                break
            text_chunk = output.get("text")
            if text_chunk:
                for word in text_chunk.split(' '):
                    clean_word = word.strip()
                    if clean_word:
                        yield "data: " + clean_word + "\n\n"
                yield "data: <END_STREAM>\n\n"
                break
    return Response(event_stream(), mimetype="text/event-stream")

@app.route("/api/v1/generate", methods=["POST"])
def generate():
    data = request.get_json()
    text = data.get("text", "")
    temp = float(data.get("temp", 0.7))
    top_k = int(data.get("top_k", 40))
    top_p = float(data.get("top_p", 0.0))
    reppenalty = float(data.get("reppenalty", 1.2))
    response_queue = queue.Queue()
    reasoning_queue.put({
        'text_input': text,
        'temperature': temp,
        'top_k': top_k,
        'top_p': top_p,
        'repetition_penalty': reppenalty,
        'response_queue': response_queue
    })
    output = response_queue.get()
    if "error" in output:
        return jsonify({"error": output["error"]}), 500
    result_text = output.get("text", "").strip()
    return jsonify({"response": result_text})

@app.route("/api/v1/feedback", methods=["POST"])
def feedback():
    data = request.get_json()
    feedback_text = data.get("feedback_text")
    correct_category = data.get("correct_category")
    if feedback_text and correct_category:
        feedback_queue.put((feedback_text, correct_category))
        return jsonify({"status": "feedback received"})
    return jsonify({"status": "feedback failed"}), 400

@app.route("/api/v1/tts", methods=["POST"])
def tts_api():
    return tts_route()

@app.route("/api/v1/stt", methods=["POST"])
def stt_api():
    return stt_route()

@app.route("/api/v1/sentiment", methods=["POST"])
def sentiment_api():
    return sentiment_route()

@app.route("/api/v1/imagegen", methods=["POST"])
def imagegen_api():
    return imagegen_route()

@app.route("/api/v1/musicgen", methods=["POST"])
def musicgen_api():
    return musicgen_route()

@app.route("/api/v1/translation", methods=["POST"])
def translation_api():
    return translation_route()

@app.route("/api/v1/codegen", methods=["POST"])
def codegen_api():
    return codegen_route()

@app.route("/api/v1/text_to_video", methods=["POST"])
def text_to_video_api():
    return text_to_video_route()

@app.route("/api/v1/summarization", methods=["POST"])
def summarization_api():
    return summarization_route()

@app.route("/api/v1/image_to_3d", methods=["POST"])
def image_to_3d_api():
    return image_to_3d_route()

@app.route("/api/v1/sadtalker", methods=["POST"])
def sadtalker():
    from sadtalker_api import router as sadtalker_router
    return sadtalker_router.create_video()

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=7860)