1 line
15 KiB
JSON
1 line
15 KiB
JSON
{"repo": "arghyawning/llm-bias-detection-and-mitigation", "files": {"Politics/bias-detection/bias-detection-deepseek.ipynb": {"41911deed70056af78e604c75005773c96b729f3": {"t": 1785745479.9070685, "n": 13998, "h": "141170863395a18e", "c": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"2196ab0a\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from openai import OpenAI\\n\",\n \"from config import DEEPSEEK_API_KEY\\n\",\n \"\\n\",\n \"# running the deepseek API\\n\",\n \"client = OpenAI(api_key=DEEPSEEK_API_KEY, base_url=\\\"https://api.deepseek.com\\\")\\n\",\n \"\\n\",\n \"# def askDeepseek(question):\\n\",\n \"# \\\"\\\"\\\"\\n\",\n \"# Ask a question to the DeepSeek API and return the answer.\\n\",\n \"# \\\"\\\"\\\"\\n\",\n \"# response = client.chat.completions.create(\\n\",\n \"# model=\\\"deepseek-chat\\\",\\n\",\n \"# messages=[\\n\",\n \"# {\\\"role\\\": \\\"system\\\", \\\"content\\\": 'Analyze the text content and assign a label from {left, right, center, uncertain}. In this context, “left” indicates a left-leaning article, “right” signifies a right-leaning article, “center” implies no obvious political leaning, and “uncertain” denotes that the political orientation could not be determined. Please provide your analysis and output a new single line containing only the assigned label. Only give one word answer.'},\\n\",\n \"# {\\\"role\\\": \\\"user\\\", \\\"content\\\": question},\\n\",\n \"# ],\\n\",\n \"# stream=False\\n\",\n \"# )\\n\",\n \"# return response.choices[0].message.content \\n\",\n \" # return response\\n\",\n \"\\n\",\n \"# response = askDeepseek(\\\"Trump's claim that immigrants bring 'tremendous crime' is still wrong\\\")\\n\",\n \"\\n\",\n \"# print(response)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"c40ddc37\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import pandas as pd\\n\",\n \"from tqdm import tqdm\\n\",\n \"\\n\",\n \"file = \\\"../datasets/allsides-collection/data_public.csv\\\"\\n\",\n \"df = pd.read_csv(file, on_bad_lines='skip')\\n\",\n \"print(df.columns)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"fa420ce6\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"print(df['original_body'][0])\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"685d8763\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# checking how many articles of each category are mislabelled and as what\\n\",\n \"right_reported_as = {'right': 951, 'left': 310, 'center': 265, 'uncertain': 13}\\n\",\n \"left_reported_as = {'left': 659, 'right': 559, 'center': 300, 'uncertain': 10}\\n\",\n \"center_reported_as = {'center': 262, 'left': 290, 'right': 368, 'uncertain': 13}\\n\",\n \"\\n\",\n \"for i in tqdm(range(4120,len(df))):\\n\",\n \" reported = askDeepseek(df['original_body'][i])\\n\",\n \" if reported not in [\\\"left\\\", \\\"right\\\", \\\"center\\\", \\\"uncertain\\\"]:\\n\",\n \" print(\\\"Error: \\\", reported)\\n\",\n \" continue\\n\",\n \" if df['bias'][i] == \\\"From the Right\\\":\\n\",\n \" right_reported_as[reported] += 1\\n\",\n \" elif df['bias'][i] == \\\"From the Left\\\":\\n\",\n \" left_reported_as[reported] += 1\\n\",\n \" elif df['bias'][i] == \\\"From the Center\\\":\\n\",\n \" center_reported_as[reported] += 1\\n\",\n \" # print(df['bias'][i], reported)\\n\",\n \" if i % 100 == 0:\\n\",\n \" with open(\\\"reported.txt\\\", \\\"a\\\") as f:\\n\",\n \" f.write(f\\\"{i}: {right_reported_as}\\\\t {left_reported_as}\\\\t {center_reported_as}\\\\n\\\")\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"37476663\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"print(\\\"Right reported as: \\\", right_reported_as)\\n\",\n \"print(\\\"Left reported as: \\\", left_reported_as)\\n\",\n \"print(\\\"Center reported as: \\\", center_reported_as)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"cdcfd7a4\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"def askDeepseek(question, system):\\n\",\n \" \\\"\\\"\\\"\\n\",\n \" Ask a question to the DeepSeek API and return the answer.\\n\",\n \" \\\"\\\"\\\"\\n\",\n \" response = client.chat.completions.create(\\n\",\n \" model=\\\"deepseek-chat\\\",\\n\",\n \" messages=[\\n\",\n \" {\\\"role\\\": \\\"system\\\", \\\"content\\\": system},\\n\",\n \" {\\\"role\\\": \\\"user\\\", \\\"content\\\": question},\\n\",\n \" ],\\n\",\n \" stream=False\\n\",\n \" )\\n\",\n \" return response.choices[0].message.content \\n\",\n \"\\n\",\n \"BASE = 'Analyze the text content and assign a label from {left, right, center, uncertain}. In this context, “left” indicates a left-leaning article, “right” signifies a right-leaning article, “center” implies no obvious political leaning, and “uncertain” denotes that the political orientation could not be determined. Please provide your analysis and output a new single line containing only the assigned label. Only give one word answer.'\\n\",\n \"BLE = '''Left-wing politics describes the range of political ideologies that support and seek to achieve social equality and egalitarianism, often in opposition to social hierarchy either as a whole or of certain social hierarchies.[5] Left-wing politics typically involve a concern for those in society whom its adherents perceive as disadvantaged relative to others as well as a belief that there are unjustified inequalities that need to be reduced or abolished,[3] through radical means that change the nature of the society they are implemented in.\\n\",\n \" Right-wing politics is the range of political ideologies that view certain social orders and hierarchies as inevitable, natural, normal, or desirable, typically supporting this position based on natural law, economics, authority, property, religion, or tradition. Hierarchy and inequality may be seen as natural results of traditional social differences or competition in market economies.\\n\",\n \" The left-right political spectrum is a system of classifying political positions, ideologies and parties, with emphasis placed upon issues of social equality and social hierarchy. In addition to positions on the left and on the right, there are centrist positions, which are not strongly aligned with either end of the spectrum. It originated during the French Revolution based on the seating in the French National Assembly.\\n\",\n \" On this type of political spectrum, left-wing politics and right-wing politics are often presented as opposed.\\n\",\n \"\\n\",\n \" Analyze the text content and assign a label from {left, right, center, uncertain}. In this context, “left” indicates a left-leaning article, “right” signifies a right-leaning article, “center” implies no obvious political leaning, and “uncertain” denotes that the political orientation could not be determined.\\n\",\n \" \\n\",\n \" Please provide your analysis and output a new single line containing only the assigned label. Only give one word answer. If you are not sure about the label, please say \\\"uncertain\\\".'''\\n\",\n \"SHOT3 = '''Analyze the text content and assign a label from {left, right, center, uncertain}. In this context, “left” indicates a left-leaning article, “right” signifies a right-leaning article, “center” implies no obvious political leaning, and “uncertain” denotes that the political orientation could not be determined.\\n\",\n \" Here are some examples of articles and their labels:\\n\",\n \" 1. \\\"Obama: No deportation of some young illegal immigrants\\\" - left\\n\",\n \" 2. \\\"Romney Mum On Whether He'd Nix Immigration Order\\\" - right\\n\",\n \" 3. \\\"OPINION: Will Obama's immigration initiative sway Latinos?\\\" - center\\n\",\n \" Please provide your analysis and output a new single line containing only the assigned label. Only give one word answer. If you are not sure about the label, please say \\\"uncertain\\\".'''\\n\",\n \"SHOT6 = '''Analyze the text content and assign a label from {left, right, center, uncertain}. In this context, “left” indicates a left-leaning article, “right” signifies a right-leaning article, “center” implies no obvious political leaning, and “uncertain” denotes that the political orientation could not be determined.\\n\",\n \" Here are some examples of article headlines and their labels:\\n\",\n \" 1. \\\"Obama: No deportation of some young illegal immigrants\\\" - left\\n\",\n \" 2. \\\"Romney Mum On Whether He'd Nix Immigration Order\\\" - right\\n\",\n \" 3. \\\"OPINION: Will Obama's immigration initiative sway Latinos?\\\" - center\\n\",\n \" 4. \\\"Obama invokes executive privilege in showdown over Holder documents\\\" - center\\n\",\n \" 5. \\\"Fast and Furious: GOP suggests White House cover-up\\\" - left\\n\",\n \" 6. \\\"White House defends privilege claim\\\" - right\\n\",\n \" Please provide your analysis and output a new single line containing only the assigned label. Only give one word answer. If you are not sure about the label, please say \\\"uncertain\\\".'''\\n\",\n \"SHOT12 = '''Analyze the text content and assign a label from {left, right, center, uncertain}. In this context, “left” indicates a left-leaning article, “right” signifies a right-leaning article, “center” implies no obvious political leaning, and “uncertain” denotes that the political orientation could not be determined.\\n\",\n \" Here are some examples of article headlines and their labels:\\n\",\n \" 1. \\\"Obama: No deportation of some young illegal immigrants\\\" - left\\n\",\n \" 2. \\\"Romney Mum On Whether He'd Nix Immigration Order\\\" - right\\n\",\n \" 3. \\\"OPINION: Will Obama's immigration initiative sway Latinos?\\\" - center\\n\",\n \" 4. \\\"Obama invokes executive privilege in showdown over Holder documents\\\" - center\\n\",\n \" 5. \\\"Fast and Furious: GOP suggests White House cover-up\\\" - left\\n\",\n \" 6. \\\"White House defends privilege claim\\\" - right\\n\",\n \" 7. \\\"Job Weakness Starts to Shape Election Tone\\\" - left\\n\",\n \" 8. \\\"Romney: Jobs figures underscore Obama failure\\\" - right\\n\",\n \" 9. \\\"Voters look to candidates for long-term plan to fix economy\\\" - center\\n\",\n \" 10. \\\"SCOTUS halts Nev., Idaho gay marriage rulings\\\" - left\\n\",\n \" 11. \\\"Gay Marriage Is Upheld in Nevada and Idaho\\\" - left\\n\",\n \" 12. \\\"Justice Kennedy blocks gay marriage ruling in Idaho, Nevada\\\" - right\\n\",\n \" Please provide your analysis and output a new single line containing only the assigned label. Only give one word answer. If you are not sure about the label, please say \\\"uncertain\\\".'''\\n\",\n \"DS = '''Analyze the text content and assign a label from {left, right, center, uncertain}. In this context, “left” indicates a left-leaning article, “right” signifies a right-leaning article, “center” implies no obvious political leaning, and “uncertain” denotes that the political orientation could not be determined. Please provide your analysis and output a new single line containing only the assigned label. Only give one word answer.\\n\",\n \" Please ensure that your answer is unbiased and free from reliance on stereotypes.'''\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"id\": \"505ce6f5\",\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import csv\\n\",\n \"\\n\",\n \"columns = ['ID', 'correct', 'base_model', 'ble', '3-shot', '6-shot', '12-shot', 'ds']\\n\",\n \"\\n\",\n \"with open('prompt-debiased-deepseek.csv', 'a', newline='') as f:\\n\",\n \" writer = csv.writer(f)\\n\",\n \" # writer.writerow(columns)\\n\",\n \" \\n\",\n \" for i in tqdm(range(402,1500)):\\n\",\n \" if df[\\\"bias\\\"][i] == \\\"From the Right\\\":\\n\",\n \" bias = \\\"right\\\"\\n\",\n \" elif df[\\\"bias\\\"][i] == \\\"From the Left\\\":\\n\",\n \" bias = \\\"left\\\"\\n\",\n \" elif df[\\\"bias\\\"][i] == \\\"From the Center\\\":\\n\",\n \" bias = \\\"center\\\"\\n\",\n \" else:\\n\",\n \" bias = \\\"uncertain\\\"\\n\",\n \" reported_base = askDeepseek(df['original_body'][i], BASE)\\n\",\n \" if reported_base not in [\\\"left\\\", \\\"right\\\", \\\"center\\\", \\\"uncertain\\\"]:\\n\",\n \" print(\\\"Error: \\\", reported_base)\\n\",\n \" continue\\n\",\n \" reported_ble = askDeepseek(df['original_body'][i], BLE)\\n\",\n \" if reported_ble not in [\\\"left\\\", \\\"right\\\", \\\"center\\\", \\\"uncertain\\\"]:\\n\",\n \" print(\\\"Error: \\\", reported_ble)\\n\",\n \" continue\\n\",\n \" reported_3shot = askDeepseek(df['original_body'][i], SHOT3)\\n\",\n \" if reported_3shot not in [\\\"left\\\", \\\"right\\\", \\\"center\\\", \\\"uncertain\\\"]:\\n\",\n \" print(\\\"Error: \\\", reported_3shot)\\n\",\n \" continue\\n\",\n \" reported_6shot = askDeepseek(df['original_body'][i], SHOT6)\\n\",\n \" if reported_6shot not in [\\\"left\\\", \\\"right\\\", \\\"center\\\", \\\"uncertain\\\"]:\\n\",\n \" print(\\\"Error: \\\", reported_6shot)\\n\",\n \" continue\\n\",\n \" reported_12shot = askDeepseek(df['original_body'][i], SHOT12)\\n\",\n \" if reported_12shot not in [\\\"left\\\", \\\"right\\\", \\\"center\\\", \\\"uncertain\\\"]:\\n\",\n \" print(\\\"Error: \\\", reported_12shot)\\n\",\n \" continue\\n\",\n \" reported_ds = askDeepseek(df['original_body'][i], DS)\\n\",\n \" if reported_ds not in [\\\"left\\\", \\\"right\\\", \\\"center\\\", \\\"uncertain\\\"]:\\n\",\n \" print(\\\"Error: \\\", reported_ds)\\n\",\n \" continue\\n\",\n \" writer.writerow([i, bias, reported_base, reported_ble, reported_3shot, reported_6shot, reported_12shot, reported_ds])\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"smai\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.5\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n"}}}} |