1 line
20 KiB
JSON
1 line
20 KiB
JSON
{"repo": "9891hubtina/meu-chatbot", "files": {"src/python/chatbot.ipynb": {"c47e608c7a88cad5bb6ba069df953da95f5dd94d": {"t": 1785745491.0138686, "n": 9093, "h": "f85b649a2c9d80f4", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# coding=utf-8\\n\",\n \"import pandas as pd\\n\",\n \"import time\\n\",\n \"import tiktoken\\n\",\n \"import requests # Para fazer requisições à API da DeepSeek\\n\",\n \"from flask import Flask, request, jsonify\\n\",\n \"import os\\n\",\n \"from scipy.spatial.distance import cosine\\n\",\n \"import numpy as np\\n\",\n \"import ast\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Inicializa o Flask\\n\",\n \"app = Flask(__name__)\\n\",\n \"\\n\",\n \"# Carrega os embeddings pré-treinados (se existirem)\\n\",\n \"dataframe = pd.DataFrame()\\n\",\n \"if 'training_embeddings.csv' in os.listdir():\\n\",\n \" dataframe = pd.read_csv('training_embeddings.csv', index_col=0)\\n\",\n \" def string_to_array(x):\\n\",\n \" return np.array(ast.literal_eval(x)) \\n\",\n \" dataframe['embedding'] = dataframe['embedding'].apply(string_to_array)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para carregar a chave da API da DeepSeek\\n\",\n \"def get_deepseek_api_key():\\n\",\n \" with open('deepseek_key.txt', 'r') as f:\\n\",\n \" return f.read().strip()\\n\",\n \"\\n\",\n \"# Carrega a chave da API da DeepSeek\\n\",\n \"deepseek_api_key = get_deepseek_api_key()\\n\",\n \"print('DeepSeek API key loaded')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para fazer uma requisição à API da DeepSeek\\n\",\n \"def query_deepseek_api(prompt):\\n\",\n \" url = \\\"https://api.deepseek.com/v1/chat/completions\\\" # Endpoint da DeepSeek\\n\",\n \" headers = {\\n\",\n \" \\\"Authorization\\\": f\\\"Bearer {deepseek_api_key}\\\",\\n\",\n \" \\\"Content-Type\\\": \\\"application/json\\\"\\n\",\n \" }\\n\",\n \" data = {\\n\",\n \" \\\"model\\\": \\\"deepseek-chat\\\", # Modelo da DeepSeek\\n\",\n \" \\\"messages\\\": [{\\\"role\\\": \\\"user\\\", \\\"content\\\": prompt}],\\n\",\n \" \\\"temperature\\\": 0.6,\\n\",\n \" \\\"max_tokens\\\": 150,\\n\",\n \" \\\"top_p\\\": 0.7,\\n\",\n \" \\\"frequency_penalty\\\": 0,\\n\",\n \" \\\"presence_penalty\\\": 0,\\n\",\n \" \\\"stop\\\": ['###']\\n\",\n \" }\\n\",\n \" response = requests.post(url, headers=headers, json=data)\\n\",\n \" return response.json()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para treinar o modelo\\n\",\n \"def train():\\n\",\n \" filename = 'training.txt'\\n\",\n \" print('Training with data from', filename)\\n\",\n \" with open(filename, 'r') as f:\\n\",\n \" data = f.read()\\n\",\n \" data = data.lower().split('\\\\n')\\n\",\n \" \\n\",\n \" dataframe = pd.DataFrame(data, columns=['text'])\\n\",\n \" dataframe.to_csv('training.csv')\\n\",\n \" print('Training data saved to training.csv')\\n\",\n \"\\n\",\n \" tokenizer = tiktoken.get_encoding(\\\"cl100k_base\\\")\\n\",\n \" dataframe = pd.read_csv('training.csv', index_col=0)\\n\",\n \" dataframe.columns = ['text']\\n\",\n \"\\n\",\n \" dataframe['n_tokens'] = dataframe['text'].apply(lambda x: len(tokenizer.encode(str(x))))\\n\",\n \"\\n\",\n \" print('Training data loaded')\\n\",\n \"\\n\",\n \" max_tokens = 512\\n\",\n \"\\n\",\n \" def split_into_many(text, max_tokens = max_tokens):\\n\",\n \" sentences = text.split(' ')\\n\",\n \" n_tokens = [len(tokenizer.encode(\\\" \\\" + sentence)) for sentence in sentences]\\n\",\n \"\\n\",\n \" chunks = []\\n\",\n \" tokens_so_far = 0\\n\",\n \" chunk = []\\n\",\n \"\\n\",\n \" for sentence, token in zip(sentences, n_tokens):\\n\",\n \" if tokens_so_far + token > max_tokens:\\n\",\n \" chunks.append(\\\". \\\".join(chunk)+ \\\".\\\")\\n\",\n \" chunk = []\\n\",\n \" tokens_so_far = 0\\n\",\n \" \\n\",\n \" if token > max_tokens:\\n\",\n \" continue\\n\",\n \"\\n\",\n \" chunk.append(sentence)\\n\",\n \" tokens_so_far += token + 1\\n\",\n \" \\n\",\n \" return chunks\\n\",\n \"\\n\",\n \" shortened = []\\n\",\n \"\\n\",\n \" for row in dataframe.iterrows():\\n\",\n \" if row[1]['text'] is None:\\n\",\n \" continue\\n\",\n \" \\n\",\n \" if row[1]['n_tokens'] > max_tokens:\\n\",\n \" shortened += split_into_many(row[1]['text'])\\n\",\n \" else:\\n\",\n \" shortened.append(row[1]['text'])\\n\",\n \"\\n\",\n \" dataframe = pd.DataFrame(shortened, columns=['text'])\\n\",\n \" dataframe['n_tokens'] = dataframe['text'].apply(lambda x: len(tokenizer.encode(str(x))))\\n\",\n \"\\n\",\n \" total_tokens = dataframe['n_tokens'].sum()\\n\",\n \" print('Total tokens:', total_tokens)\\n\",\n \" print('total embedding cost:', total_tokens/1000 * 0.0001)\\n\",\n \"\\n\",\n \" i = 0\\n\",\n \" embeddings = []\\n\",\n \" for text in dataframe['text']:\\n\",\n \" try:\\n\",\n \" print('Creating embedding', i)\\n\",\n \" embedding = openai.Embedding.create(input=text, engine='text-embedding-ada-002')['data'][0]['embedding']\\n\",\n \" embeddings.append(embedding)\\n\",\n \" time.sleep(1)\\n\",\n \" except openai.error.RateLimitError:\\n\",\n \" print('Rate limit reached, sleeping for 1 minute')\\n\",\n \" time.sleep(20)\\n\",\n \" embedding = openai.Embedding.create(input=text, engine='text-embedding-ada-002')['data'][0]['embedding']\\n\",\n \" embeddings.append(embedding)\\n\",\n \" i += 1\\n\",\n \" \\n\",\n \" print('Embeddings created')\\n\",\n \" dataframe['embedding'] = embeddings\\n\",\n \" dataframe.to_csv('training_embeddings.csv')\\n\",\n \" print('Embeddings saved to training_embeddings.csv')\\n\",\n \" print('Training complete')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para criar contexto\\n\",\n \"def create_context(question, df, max_len=1800, size=\\\"ada\\\"):\\n\",\n \" print('Creating context for question:', question)\\n\",\n \" q_embeddings = openai.Embedding.create(input=question, engine='text-embedding-ada-002')['data'][0]['embedding']\\n\",\n \" df['distances'] = df['embedding'].apply(lambda x: cosine(q_embeddings, x))\\n\",\n \" returns = []\\n\",\n \" cur_len = 0\\n\",\n \" for i, row in df.sort_values('distances', ascending=True).iterrows():\\n\",\n \" cur_len += row['n_tokens'] + 4\\n\",\n \" if cur_len > max_len:\\n\",\n \" break\\n\",\n \" returns.append(row['text'])\\n\",\n \" return \\\"\\\\n###\\\\n\\\".join(returns)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para responder perguntas usando a API da DeepSeek\\n\",\n \"def answer(question):\\n\",\n \" context = create_context(question, dataframe)\\n\",\n \" prompt = f\\\"Você é um assistente para o site da faculdade, seja profissional e responda as perguntas baseado no contexto abaixo, recuse educadamente caso não possa responder.\\\\n{context}\\\\n---\\\\nPergunta: {question}\\\\nResposta:\\\"\\n\",\n \" \\n\",\n \" try:\\n\",\n \" response = query_deepseek_api(prompt)\\n\",\n \" return response['choices'][0]['message']['content']\\n\",\n \" except Exception as e:\\n\",\n \" print(e)\\n\",\n \" return 'Minha cabeça está doendo, não consigo responder a essa pergunta agora.'\\n\",\n \" \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Endpoint para responder perguntas\\n\",\n \"@app.route('/answer', methods=['POST'])\\n\",\n \"def get_answer():\\n\",\n \" question = request.json['question']\\n\",\n \" return jsonify({'answer': answer(question)})\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Endpoint para treinar o modelo\\n\",\n \"@app.route('/train', methods=['POST'])\\n\",\n \"def train_model():\\n\",\n \" train()\\n\",\n \" return jsonify({'status': 'Training complete'})\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Inicia o servidor Flask\\n\",\n \"if __name__ == '__main__':\\n\",\n \" app.run(host='0.0.0.0', port=2024)\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.12.7\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"}, "409a5c439848a0968b7fcab9e7d2ac54ddb1896e": {"t": 1785753692.1451528, "n": 9093, "h": "f85b649a2c9d80f4", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# coding=utf-8\\n\",\n \"import pandas as pd\\n\",\n \"import time\\n\",\n \"import tiktoken\\n\",\n \"import requests # Para fazer requisições à API da DeepSeek\\n\",\n \"from flask import Flask, request, jsonify\\n\",\n \"import os\\n\",\n \"from scipy.spatial.distance import cosine\\n\",\n \"import numpy as np\\n\",\n \"import ast\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Inicializa o Flask\\n\",\n \"app = Flask(__name__)\\n\",\n \"\\n\",\n \"# Carrega os embeddings pré-treinados (se existirem)\\n\",\n \"dataframe = pd.DataFrame()\\n\",\n \"if 'training_embeddings.csv' in os.listdir():\\n\",\n \" dataframe = pd.read_csv('training_embeddings.csv', index_col=0)\\n\",\n \" def string_to_array(x):\\n\",\n \" return np.array(ast.literal_eval(x)) \\n\",\n \" dataframe['embedding'] = dataframe['embedding'].apply(string_to_array)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para carregar a chave da API da DeepSeek\\n\",\n \"def get_deepseek_api_key():\\n\",\n \" with open('deepseek_key.txt', 'r') as f:\\n\",\n \" return f.read().strip()\\n\",\n \"\\n\",\n \"# Carrega a chave da API da DeepSeek\\n\",\n \"deepseek_api_key = get_deepseek_api_key()\\n\",\n \"print('DeepSeek API key loaded')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para fazer uma requisição à API da DeepSeek\\n\",\n \"def query_deepseek_api(prompt):\\n\",\n \" url = \\\"https://api.deepseek.com/v1/chat/completions\\\" # Endpoint da DeepSeek\\n\",\n \" headers = {\\n\",\n \" \\\"Authorization\\\": f\\\"Bearer {deepseek_api_key}\\\",\\n\",\n \" \\\"Content-Type\\\": \\\"application/json\\\"\\n\",\n \" }\\n\",\n \" data = {\\n\",\n \" \\\"model\\\": \\\"deepseek-chat\\\", # Modelo da DeepSeek\\n\",\n \" \\\"messages\\\": [{\\\"role\\\": \\\"user\\\", \\\"content\\\": prompt}],\\n\",\n \" \\\"temperature\\\": 0.6,\\n\",\n \" \\\"max_tokens\\\": 150,\\n\",\n \" \\\"top_p\\\": 0.7,\\n\",\n \" \\\"frequency_penalty\\\": 0,\\n\",\n \" \\\"presence_penalty\\\": 0,\\n\",\n \" \\\"stop\\\": ['###']\\n\",\n \" }\\n\",\n \" response = requests.post(url, headers=headers, json=data)\\n\",\n \" return response.json()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para treinar o modelo\\n\",\n \"def train():\\n\",\n \" filename = 'training.txt'\\n\",\n \" print('Training with data from', filename)\\n\",\n \" with open(filename, 'r') as f:\\n\",\n \" data = f.read()\\n\",\n \" data = data.lower().split('\\\\n')\\n\",\n \" \\n\",\n \" dataframe = pd.DataFrame(data, columns=['text'])\\n\",\n \" dataframe.to_csv('training.csv')\\n\",\n \" print('Training data saved to training.csv')\\n\",\n \"\\n\",\n \" tokenizer = tiktoken.get_encoding(\\\"cl100k_base\\\")\\n\",\n \" dataframe = pd.read_csv('training.csv', index_col=0)\\n\",\n \" dataframe.columns = ['text']\\n\",\n \"\\n\",\n \" dataframe['n_tokens'] = dataframe['text'].apply(lambda x: len(tokenizer.encode(str(x))))\\n\",\n \"\\n\",\n \" print('Training data loaded')\\n\",\n \"\\n\",\n \" max_tokens = 512\\n\",\n \"\\n\",\n \" def split_into_many(text, max_tokens = max_tokens):\\n\",\n \" sentences = text.split(' ')\\n\",\n \" n_tokens = [len(tokenizer.encode(\\\" \\\" + sentence)) for sentence in sentences]\\n\",\n \"\\n\",\n \" chunks = []\\n\",\n \" tokens_so_far = 0\\n\",\n \" chunk = []\\n\",\n \"\\n\",\n \" for sentence, token in zip(sentences, n_tokens):\\n\",\n \" if tokens_so_far + token > max_tokens:\\n\",\n \" chunks.append(\\\". \\\".join(chunk)+ \\\".\\\")\\n\",\n \" chunk = []\\n\",\n \" tokens_so_far = 0\\n\",\n \" \\n\",\n \" if token > max_tokens:\\n\",\n \" continue\\n\",\n \"\\n\",\n \" chunk.append(sentence)\\n\",\n \" tokens_so_far += token + 1\\n\",\n \" \\n\",\n \" return chunks\\n\",\n \"\\n\",\n \" shortened = []\\n\",\n \"\\n\",\n \" for row in dataframe.iterrows():\\n\",\n \" if row[1]['text'] is None:\\n\",\n \" continue\\n\",\n \" \\n\",\n \" if row[1]['n_tokens'] > max_tokens:\\n\",\n \" shortened += split_into_many(row[1]['text'])\\n\",\n \" else:\\n\",\n \" shortened.append(row[1]['text'])\\n\",\n \"\\n\",\n \" dataframe = pd.DataFrame(shortened, columns=['text'])\\n\",\n \" dataframe['n_tokens'] = dataframe['text'].apply(lambda x: len(tokenizer.encode(str(x))))\\n\",\n \"\\n\",\n \" total_tokens = dataframe['n_tokens'].sum()\\n\",\n \" print('Total tokens:', total_tokens)\\n\",\n \" print('total embedding cost:', total_tokens/1000 * 0.0001)\\n\",\n \"\\n\",\n \" i = 0\\n\",\n \" embeddings = []\\n\",\n \" for text in dataframe['text']:\\n\",\n \" try:\\n\",\n \" print('Creating embedding', i)\\n\",\n \" embedding = openai.Embedding.create(input=text, engine='text-embedding-ada-002')['data'][0]['embedding']\\n\",\n \" embeddings.append(embedding)\\n\",\n \" time.sleep(1)\\n\",\n \" except openai.error.RateLimitError:\\n\",\n \" print('Rate limit reached, sleeping for 1 minute')\\n\",\n \" time.sleep(20)\\n\",\n \" embedding = openai.Embedding.create(input=text, engine='text-embedding-ada-002')['data'][0]['embedding']\\n\",\n \" embeddings.append(embedding)\\n\",\n \" i += 1\\n\",\n \" \\n\",\n \" print('Embeddings created')\\n\",\n \" dataframe['embedding'] = embeddings\\n\",\n \" dataframe.to_csv('training_embeddings.csv')\\n\",\n \" print('Embeddings saved to training_embeddings.csv')\\n\",\n \" print('Training complete')\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para criar contexto\\n\",\n \"def create_context(question, df, max_len=1800, size=\\\"ada\\\"):\\n\",\n \" print('Creating context for question:', question)\\n\",\n \" q_embeddings = openai.Embedding.create(input=question, engine='text-embedding-ada-002')['data'][0]['embedding']\\n\",\n \" df['distances'] = df['embedding'].apply(lambda x: cosine(q_embeddings, x))\\n\",\n \" returns = []\\n\",\n \" cur_len = 0\\n\",\n \" for i, row in df.sort_values('distances', ascending=True).iterrows():\\n\",\n \" cur_len += row['n_tokens'] + 4\\n\",\n \" if cur_len > max_len:\\n\",\n \" break\\n\",\n \" returns.append(row['text'])\\n\",\n \" return \\\"\\\\n###\\\\n\\\".join(returns)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Função para responder perguntas usando a API da DeepSeek\\n\",\n \"def answer(question):\\n\",\n \" context = create_context(question, dataframe)\\n\",\n \" prompt = f\\\"Você é um assistente para o site da faculdade, seja profissional e responda as perguntas baseado no contexto abaixo, recuse educadamente caso não possa responder.\\\\n{context}\\\\n---\\\\nPergunta: {question}\\\\nResposta:\\\"\\n\",\n \" \\n\",\n \" try:\\n\",\n \" response = query_deepseek_api(prompt)\\n\",\n \" return response['choices'][0]['message']['content']\\n\",\n \" except Exception as e:\\n\",\n \" print(e)\\n\",\n \" return 'Minha cabeça está doendo, não consigo responder a essa pergunta agora.'\\n\",\n \" \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Endpoint para responder perguntas\\n\",\n \"@app.route('/answer', methods=['POST'])\\n\",\n \"def get_answer():\\n\",\n \" question = request.json['question']\\n\",\n \" return jsonify({'answer': answer(question)})\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Endpoint para treinar o modelo\\n\",\n \"@app.route('/train', methods=['POST'])\\n\",\n \"def train_model():\\n\",\n \" train()\\n\",\n \" return jsonify({'status': 'Training complete'})\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Inicia o servidor Flask\\n\",\n \"if __name__ == '__main__':\\n\",\n \" app.run(host='0.0.0.0', port=2024)\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.12.7\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n"}}}} |