|
from llama_cpp import Llama |
|
from huggingface_hub import hf_hub_download |
|
from flask import Flask, request, jsonify |
|
|
|
app = Flask(__name__) |
|
|
|
hf_hub_download("TheBloke/phi-2-GGUF", "phi-2.Q8_0.gguf", local_dir="./") |
|
phi = Llama(model_path="./phi-2.Q8_0.gguf", n_ctx=2048, n_gpu_layers=999) |
|
|
|
app.route("/spaces/MrOvkill/phastfi/", methods=["GET"]) |
|
def index(): |
|
return "<html><body><h1>Use API</h1><p>Use /completion as POST with a prompt in a JSON query.</p></body></html>" |
|
|
|
@app.route("/spaces/MrOvkill/phastfi/generate", methods=["POST"]) |
|
def completion(): |
|
prompt = request.json["prompt"] |
|
res = phi( |
|
prompt, |
|
temperature=0.33, |
|
top_p=0.95, |
|
top_k=42, |
|
max_tokens=1024, |
|
num_completions=2, |
|
) |
|
return jsonify({ |
|
"responses": [ |
|
res["choices"][0]["text"], |
|
res["choices"][1]["text"] |
|
] |
|
}) |
|
|
|
if __name__ == "__main__": |
|
app.run(host="0.0.0.0", port=7860) |
