{"repo": "boyiwei/cotaeval", "files": {"eval/FastChat_new/fastchat/serve/gradio_web_server.py": {"e5aefd81f5c7b088b3a49e1961b461a87aee3ae5": {"t": 1785674494.787392, "n": 33407, "h": "75dbce117bef7f70", "c": "\"\"\"\nThe gradio demo server for chatting with a single model.\n\"\"\"\n\nimport argparse\nfrom collections import defaultdict\nimport datetime\nimport hashlib\nimport json\nimport os\nimport random\nimport time\nimport uuid\n\nimport gradio as gr\nimport requests\n\nfrom fastchat.constants import (\n    LOGDIR,\n    WORKER_API_TIMEOUT,\n    ErrorCode,\n    MODERATION_MSG,\n    CONVERSATION_LIMIT_MSG,\n    RATE_LIMIT_MSG,\n    SERVER_ERROR_MSG,\n    INPUT_CHAR_LEN_LIMIT,\n    CONVERSATION_TURN_LIMIT,\n    SESSION_EXPIRATION_TIME,\n)\nfrom fastchat.model.model_adapter import (\n    get_conversation_template,\n)\nfrom fastchat.model.model_registry import get_model_info, model_info\nfrom fastchat.serve.api_provider import get_api_provider_stream_iter\nfrom fastchat.serve.remote_logger import get_remote_logger\nfrom fastchat.utils import (\n    build_logger,\n    get_window_url_params_js,\n    get_window_url_params_with_tos_js,\n    moderation_filter,\n    parse_gradio_auth_creds,\n    load_image,\n)\n\nlogger = build_logger(\"gradio_web_server\", \"gradio_web_server.log\")\n\nheaders = {\"User-Agent\": \"FastChat Client\"}\n\nno_change_btn = gr.Button()\nenable_btn = gr.Button(interactive=True, visible=True)\ndisable_btn = gr.Button(interactive=False)\ninvisible_btn = gr.Button(interactive=False, visible=False)\n\ncontroller_url = None\nenable_moderation = False\nuse_remote_storage = False\n\nacknowledgment_md = \"\"\"\n### Terms of Service\n\nUsers are required to agree to the following terms before using the service:\n\nThe service is a research preview. It only provides limited safety measures and may generate offensive content.\nIt must not be used for any illegal, harmful, violent, racist, or sexual purposes.\nPlease do not upload any private information.\nThe service collects user dialogue data, including both text and images, and reserves the right to distribute it under a Creative Commons Attribution (CC-BY) or a similar license.\n\n### Acknowledgment\nWe thank [UC Berkeley SkyLab](https://sky.cs.berkeley.edu/), [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Hyperbolic](https://hyperbolic.xyz/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous [sponsorship](https://lmsys.org/donations/).\n\n<div class=\"sponsor-image-about\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/skylab.png\" alt=\"SkyLab\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/kaggle.png\" alt=\"Kaggle\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/mbzuai.jpeg\" alt=\"MBZUAI\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/a16z.jpeg\" alt=\"a16z\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/together.png\" alt=\"Together AI\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/hyperbolic_logo.png\" alt=\"Hyperbolic\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/anyscale.png\" alt=\"AnyScale\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/huggingface.png\" alt=\"HuggingFace\">\n</div>\n\"\"\"\n\n# JSON file format of API-based models:\n# {\n#   \"gpt-3.5-turbo\": {\n#     \"model_name\": \"gpt-3.5-turbo\",\n#     \"api_type\": \"openai\",\n#     \"api_base\": \"https://api.openai.com/v1\",\n#     \"api_key\": \"sk-******\",\n#     \"anony_only\": false\n#   }\n# }\n#\n#  - \"api_type\" can be one of the following: openai, anthropic, gemini, or mistral. For custom APIs, add a new type and implement it accordingly.\n#  - \"anony_only\" indicates whether to display this model in anonymous mode only.\n\napi_endpoint_info = {}\n\n\nclass State:\n    def __init__(self, model_name, is_vision=False):\n        self.conv = get_conversation_template(model_name)\n        self.conv_id = uuid.uuid4().hex\n        self.skip_next = False\n        self.model_name = model_name\n        self.oai_thread_id = None\n        self.is_vision = is_vision\n\n        # NOTE(chris): This could be sort of a hack since it assumes the user only uploads one image. If they can upload multiple, we should store a list of image hashes.\n        self.has_csam_image = False\n\n        self.regen_support = True\n        if \"browsing\" in model_name:\n            self.regen_support = False\n        self.init_system_prompt(self.conv)\n\n    def init_system_prompt(self, conv):\n        system_prompt = conv.get_system_message()\n        if len(system_prompt) == 0:\n            return\n        current_date = datetime.datetime.now().strftime(\"%Y-%m-%d\")\n        system_prompt = system_prompt.replace(\"{{currentDateTime}}\", current_date)\n        conv.set_system_message(system_prompt)\n\n    def to_gradio_chatbot(self):\n        return self.conv.to_gradio_chatbot()\n\n    def dict(self):\n        base = self.conv.dict()\n        base.update(\n            {\n                \"conv_id\": self.conv_id,\n                \"model_name\": self.model_name,\n                \"has_csam_image\": self.has_csam_image,\n            }\n        )\n        return base\n\n\ndef set_global_vars(controller_url_, enable_moderation_, use_remote_storage_):\n    global controller_url, enable_moderation, use_remote_storage\n    controller_url = controller_url_\n    enable_moderation = enable_moderation_\n    use_remote_storage = use_remote_storage_\n\n\ndef get_conv_log_filename(is_vision=False):\n    t = datetime.datetime.now()\n    conv_log_filename = f\"{t.year}-{t.month:02d}-{t.day:02d}-conv.json\"\n    if is_vision:\n        name = os.path.join(LOGDIR, f\"vision-tmp-{conv_log_filename}\")\n    else:\n        name = os.path.join(LOGDIR, conv_log_filename)\n\n    return name\n\n\ndef get_model_list(controller_url, register_api_endpoint_file, vision_arena):\n    global api_endpoint_info\n\n    # Add models from the controller\n    if controller_url:\n        ret = requests.post(controller_url + \"/refresh_all_workers\")\n        assert ret.status_code == 200\n\n        if vision_arena:\n            ret = requests.post(controller_url + \"/list_multimodal_models\")\n            models = ret.json()[\"models\"]\n        else:\n            ret = requests.post(controller_url + \"/list_language_models\")\n            models = ret.json()[\"models\"]\n    else:\n        models = []\n\n    # Add models from the API providers\n    if register_api_endpoint_file:\n        api_endpoint_info = json.load(open(register_api_endpoint_file))\n        for mdl, mdl_dict in api_endpoint_info.items():\n            mdl_vision = mdl_dict.get(\"vision-arena\", False)\n            mdl_text = mdl_dict.get(\"text-arena\", True)\n            if vision_arena and mdl_vision:\n                models.append(mdl)\n            if not vision_arena and mdl_text:\n                models.append(mdl)\n\n    # Remove anonymous models\n    models = list(set(models))\n    visible_models = models.copy()\n    for mdl in models:\n        if mdl not in api_endpoint_info:\n            continue\n        mdl_dict = api_endpoint_info[mdl]\n        if mdl_dict[\"anony_only\"]:\n            visible_models.remove(mdl)\n\n    # Sort models and add descriptions\n    priority = {k: f\"___{i:03d}\" for i, k in enumerate(model_info)}\n    models.sort(key=lambda x: priority.get(x, x))\n    visible_models.sort(key=lambda x: priority.get(x, x))\n    logger.info(f\"All models: {models}\")\n    logger.info(f\"Visible models: {visible_models}\")\n    return visible_models, models\n\n\ndef load_demo_single(models, url_params):\n    selected_model = models[0] if len(models) > 0 else \"\"\n    if \"model\" in url_params:\n        model = url_params[\"model\"]\n        if model in models:\n            selected_model = model\n\n    dropdown_update = gr.Dropdown(choices=models, value=selected_model, visible=True)\n    state = None\n    return state, dropdown_update\n\n\ndef load_demo(url_params, request: gr.Request):\n    global models\n\n    ip = get_ip(request)\n    logger.info(f\"load_demo. ip: {ip}. params: {url_params}\")\n\n    if args.model_list_mode == \"reload\":\n        models, all_models = get_model_list(\n            controller_url, args.register_api_endpoint_file, vision_arena=False\n        )\n\n    return load_demo_single(models, url_params)\n\n\ndef vote_last_response(state, vote_type, model_selector, request: gr.Request):\n    filename = get_conv_log_filename()\n    if \"llava\" in model_selector:\n        filename = filename.replace(\"2024\", \"vision-tmp-2024\")\n\n    with open(filename, \"a\") as fout:\n        data = {\n            \"tstamp\": round(time.time(), 4),\n            \"type\": vote_type,\n            \"model\": model_selector,\n            \"state\": state.dict(),\n            \"ip\": get_ip(request),\n        }\n        fout.write(json.dumps(data) + \"\\n\")\n    get_remote_logger().log(data)\n\n\ndef upvote_last_response(state, model_selector, request: gr.Request):\n    ip = get_ip(request)\n    logger.info(f\"upvote. ip: {ip}\")\n    vote_last_response(state, \"upvote\", model_selector, request)\n    return (\"\",) + (disable_btn,) * 3\n\n\ndef downvote_last_response(state, model_selector, request: gr.Request):\n    ip = get_ip(request)\n    logger.info(f\"downvote. ip: {ip}\")\n    vote_last_response(state, \"downvote\", model_selector, request)\n    return (\"\",) + (disable_btn,) * 3\n\n\ndef flag_last_response(state, model_selector, request: gr.Request):\n    ip = get_ip(request)\n    logger.info(f\"flag. ip: {ip}\")\n    vote_last_response(state, \"flag\", model_selector, request)\n    return (\"\",) + (disable_btn,) * 3\n\n\ndef regenerate(state, request: gr.Request):\n    ip = get_ip(request)\n    logger.info(f\"regenerate. ip: {ip}\")\n    if not state.regen_support:\n        state.skip_next = True\n        return (state, state.to_gradio_chatbot(), \"\", None) + (no_change_btn,) * 5\n    state.conv.update_last_message(None)\n    return (state, state.to_gradio_chatbot(), \"\", None) + (disable_btn,) * 5\n\n\ndef clear_history(request: gr.Request):\n    ip = get_ip(request)\n    logger.info(f\"clear_history. ip: {ip}\")\n    state = None\n    return (state, [], \"\", None) + (disable_btn,) * 5\n\n\ndef get_ip(request: gr.Request):\n    if \"cf-connecting-ip\" in request.headers:\n        ip = request.headers[\"cf-connecting-ip\"]\n    elif \"x-forwarded-for\" in request.headers:\n        ip = request.headers[\"x-forwarded-for\"]\n    else:\n        ip = request.client.host\n    return ip\n\n\n# TODO(Chris): At some point, we would like this to be a live-reporting feature.\ndef report_csam_image(state, image):\n    pass\n\n\ndef _prepare_text_with_image(state, text, images, csam_flag):\n    if images is not None and len(images) > 0:\n        image = images[0]\n\n        if len(state.conv.get_images()) > 0:\n            # reset convo with new image\n            state.conv = get_conversation_template(state.model_name)\n\n        resize_image = \"llava\" in state.model_name\n        image = state.conv.convert_image_to_base64(\n            image,\n            resize_image=resize_image,\n        )  # PIL type is not JSON serializable\n\n        if csam_flag:\n            state.has_csam_image = True\n            report_csam_image(state, image)\n\n        text = text, [image]\n\n    return text\n\n\ndef add_text(state, model_selector, text, image, request: gr.Request):\n    ip = get_ip(request)\n    logger.info(f\"add_text. ip: {ip}. len: {len(text)}\")\n\n    if state is None:\n        state = State(model_selector)\n\n    if len(text) <= 0:\n        state.skip_next = True\n        return (state, state.to_gradio_chatbot(), \"\", None) + (no_change_btn,) * 5\n\n    all_conv_text = state.conv.get_prompt()\n    all_conv_text = all_conv_text[-2000:] + \"\\nuser: \" + text\n    flagged = moderation_filter(all_conv_text, [state.model_name])\n    # flagged = moderation_filter(text, [state.model_name])\n    if flagged:\n        logger.info(f\"violate moderation. ip: {ip}. text: {text}\")\n        # overwrite the original text\n        text = MODERATION_MSG\n\n    if (len(state.conv.messages) - state.conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:\n        logger.info(f\"conversation turn limit. ip: {ip}. text: {text}\")\n        state.skip_next = True\n        return (state, state.to_gradio_chatbot(), CONVERSATION_LIMIT_MSG, None) + (\n            no_change_btn,\n        ) * 5\n\n    text = text[:INPUT_CHAR_LEN_LIMIT]  # Hard cut-off\n    text = _prepare_text_with_image(state, text, image, csam_flag=False)\n    state.conv.append_message(state.conv.roles[0], text)\n    state.conv.append_message(state.conv.roles[1], None)\n    return (state, state.to_gradio_chatbot(), \"\", None) + (disable_btn,) * 5\n\n\ndef model_worker_stream_iter(\n    conv,\n    model_name,\n    worker_addr,\n    prompt,\n    temperature,\n    repetition_penalty,\n    top_p,\n    max_new_tokens,\n    images,\n):\n    # Make requests\n    gen_params = {\n        \"model\": model_name,\n        \"prompt\": prompt,\n        \"temperature\": temperature,\n        \"repetition_penalty\": repetition_penalty,\n        \"top_p\": top_p,\n        \"max_new_tokens\": max_new_tokens,\n        \"stop\": conv.stop_str,\n        \"stop_token_ids\": conv.stop_token_ids,\n        \"echo\": False,\n    }\n\n    logger.info(f\"==== request ====\\n{gen_params}\")\n\n    if len(images) > 0:\n        gen_params[\"images\"] = images\n\n    # Stream output\n    response = requests.post(\n        worker_addr + \"/worker_generate_stream\",\n        headers=headers,\n        json=gen_params,\n        stream=True,\n        timeout=WORKER_API_TIMEOUT,\n    )\n    for chunk in response.iter_lines(decode_unicode=False, delimiter=b\"\\0\"):\n        if chunk:\n            data = json.loads(chunk.decode())\n            yield data\n\n\ndef is_limit_reached(model_name, ip):\n    monitor_url = \"http://localhost:9090\"\n    try:\n        ret = requests.get(\n            f\"{monitor_url}/is_limit_reached?model={model_name}&user_id={ip}\", timeout=1\n        )\n        obj = ret.json()\n        return obj\n    except Exception as e:\n        logger.info(f\"monitor error: {e}\")\n        return None\n\n\ndef bot_response(\n    state,\n    temperature,\n    top_p,\n    max_new_tokens,\n    request: gr.Request,\n    apply_rate_limit=True,\n    use_recommended_config=False,\n):\n    ip = get_ip(request)\n    logger.info(f\"bot_response. ip: {ip}\")\n    start_tstamp = time.time()\n    temperature = float(temperature)\n    top_p = float(top_p)\n    max_new_tokens = int(max_new_tokens)\n\n    if state.skip_next:\n        # This generate call is skipped due to invalid inputs\n        state.skip_next = False\n        yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5\n        return\n\n    if apply_rate_limit:\n        ret = is_limit_reached(state.model_name, ip)\n        if ret is not None and ret[\"is_limit_reached\"]:\n            error_msg = RATE_LIMIT_MSG + \"\\n\\n\" + ret[\"reason\"]\n            logger.info(f\"rate limit reached. ip: {ip}. error_msg: {ret['reason']}\")\n            state.conv.update_last_message(error_msg)\n            yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5\n            return\n\n    conv, model_name = state.conv, state.model_name\n    model_api_dict = (\n        api_endpoint_info[model_name] if model_name in api_endpoint_info else None\n    )\n    images = conv.get_images()\n\n    if model_api_dict is None:\n        # Query worker address\n        ret = requests.post(\n            controller_url + \"/get_worker_address\", json={\"model\": model_name}\n        )\n        worker_addr = ret.json()[\"address\"]\n        logger.info(f\"model_name: {model_name}, worker_addr: {worker_addr}\")\n\n        # No available worker\n        if worker_addr == \"\":\n            conv.update_last_message(SERVER_ERROR_MSG)\n            yield (\n                state,\n                state.to_gradio_chatbot(),\n                disable_btn,\n                disable_btn,\n                disable_btn,\n                enable_btn,\n                enable_btn,\n            )\n            return\n\n        # Construct prompt.\n        # We need to call it here, so it will not be affected by \"▌\".\n        prompt = conv.get_prompt()\n        # Set repetition_penalty\n        if \"t5\" in model_name:\n            repetition_penalty = 1.2\n        else:\n            repetition_penalty = 1.0\n\n        stream_iter = model_worker_stream_iter(\n            conv,\n            model_name,\n            worker_addr,\n            prompt,\n            temperature,\n            repetition_penalty,\n            top_p,\n            max_new_tokens,\n            images,\n        )\n    else:\n        if use_recommended_config:\n            recommended_config = model_api_dict.get(\"recommended_config\", None)\n            if recommended_config is not None:\n                temperature = recommended_config.get(\"temperature\", temperature)\n                top_p = recommended_config.get(\"top_p\", top_p)\n                max_new_tokens = recommended_config.get(\n                    \"max_new_tokens\", max_new_tokens\n                )\n\n        stream_iter = get_api_provider_stream_iter(\n            conv,\n            model_name,\n            model_api_dict,\n            temperature,\n            top_p,\n            max_new_tokens,\n            state,\n        )\n\n    html_code = ' <span class=\"cursor\"></span> '\n\n    # conv.update_last_message(\"▌\")\n    conv.update_last_message(html_code)\n    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5\n\n    try:\n        data = {\"text\": \"\"}\n        for i, data in enumerate(stream_iter):\n            if data[\"error_code\"] == 0:\n                output = data[\"text\"].strip()\n                # conv.update_last_message(output + \"▌\")\n                conv.update_last_message(output + html_code)\n                yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5\n            else:\n                output = data[\"text\"] + f\"\\n\\n(error_code: {data['error_code']})\"\n                conv.update_last_message(output)\n                yield (state, state.to_gradio_chatbot()) + (\n                    disable_btn,\n                    disable_btn,\n                    disable_btn,\n                    enable_btn,\n                    enable_btn,\n                )\n                return\n        output = data[\"text\"].strip()\n        conv.update_last_message(output)\n        yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5\n    except requests.exceptions.RequestException as e:\n        conv.update_last_message(\n            f\"{SERVER_ERROR_MSG}\\n\\n\"\n            f\"(error_code: {ErrorCode.GRADIO_REQUEST_ERROR}, {e})\"\n        )\n        yield (state, state.to_gradio_chatbot()) + (\n            disable_btn,\n            disable_btn,\n            disable_btn,\n            enable_btn,\n            enable_btn,\n        )\n        return\n    except Exception as e:\n        conv.update_last_message(\n            f\"{SERVER_ERROR_MSG}\\n\\n\"\n            f\"(error_code: {ErrorCode.GRADIO_STREAM_UNKNOWN_ERROR}, {e})\"\n        )\n        yield (state, state.to_gradio_chatbot()) + (\n            disable_btn,\n            disable_btn,\n            disable_btn,\n            enable_btn,\n            enable_btn,\n        )\n        return\n\n    finish_tstamp = time.time()\n    logger.info(f\"{output}\")\n\n    conv.save_new_images(\n        has_csam_images=state.has_csam_image, use_remote_storage=use_remote_storage\n    )\n\n    filename = get_conv_log_filename(is_vision=state.is_vision)\n\n    with open(filename, \"a\") as fout:\n        data = {\n            \"tstamp\": round(finish_tstamp, 4),\n            \"type\": \"chat\",\n            \"model\": model_name,\n            \"gen_params\": {\n                \"temperature\": temperature,\n                \"top_p\": top_p,\n                \"max_new_tokens\": max_new_tokens,\n            },\n            \"start\": round(start_tstamp, 4),\n            \"finish\": round(finish_tstamp, 4),\n            \"state\": state.dict(),\n            \"ip\": get_ip(request),\n        }\n        fout.write(json.dumps(data) + \"\\n\")\n    get_remote_logger().log(data)\n\n\nblock_css = \"\"\"\n#notice_markdown .prose {\n    font-size: 110% !important;\n}\n#notice_markdown th {\n    display: none;\n}\n#notice_markdown td {\n    padding-top: 6px;\n    padding-bottom: 6px;\n}\n#arena_leaderboard_dataframe table {\n    font-size: 110%;\n}\n#full_leaderboard_dataframe table {\n    font-size: 110%;\n}\n#model_description_markdown {\n    font-size: 110% !important;\n}\n#leaderboard_markdown .prose {\n    font-size: 110% !important;\n}\n#leaderboard_markdown td {\n    padding-top: 6px;\n    padding-bottom: 6px;\n}\n#leaderboard_dataframe td {\n    line-height: 0.1em;\n}\n#about_markdown .prose {\n    font-size: 110% !important;\n}\n#ack_markdown .prose {\n    font-size: 110% !important;\n}\n#chatbot .prose {\n    font-size: 105% !important;\n}\n.sponsor-image-about img {\n    margin: 0 20px;\n    margin-top: 20px;\n    height: 40px;\n    max-height: 100%;\n    width: auto;\n    float: left;\n}\n\n.chatbot h1, h2, h3 {\n    margin-top: 8px; /* Adjust the value as needed */\n    margin-bottom: 0px; /* Adjust the value as needed */\n    padding-bottom: 0px;\n}\n\n.chatbot h1 {\n    font-size: 130%;\n}\n.chatbot h2 {\n    font-size: 120%;\n}\n.chatbot h3 {\n    font-size: 110%;\n}\n.chatbot p:not(:first-child) {\n    margin-top: 8px;\n}\n\n.typing {\n    display: inline-block;\n}\n\n.cursor {\n    display: inline-block;\n    width: 7px;\n    height: 1em;\n    background-color: black;\n    vertical-align: middle;\n    animation: blink 1s infinite;\n}\n\n.dark .cursor {\n    display: inline-block;\n    width: 7px;\n    height: 1em;\n    background-color: white;\n    vertical-align: middle;\n    animation: blink 1s infinite;\n}\n\n@keyframes blink {\n    0%, 50% { opacity: 1; }\n    50.1%, 100% { opacity: 0; }\n}\n\n.app {\n  max-width: 100% !important;\n  padding: 20px !important;               \n}\n\na {\n    color: #1976D2; /* Your current link color, a shade of blue */\n    text-decoration: none; /* Removes underline from links */\n}\na:hover {\n    color: #63A4FF; /* This can be any color you choose for hover */\n    text-decoration: underline; /* Adds underline on hover */\n}\n\"\"\"\n\n\ndef get_model_description_md(models):\n    model_description_md = \"\"\"\n| | | |\n| ---- | ---- | ---- |\n\"\"\"\n    ct = 0\n    visited = set()\n    for i, name in enumerate(models):\n        minfo = get_model_info(name)\n        if minfo.simple_name in visited:\n            continue\n        visited.add(minfo.simple_name)\n        one_model_md = f\"[{minfo.simple_name}]({minfo.link}): {minfo.description}\"\n\n        if ct % 3 == 0:\n            model_description_md += \"|\"\n        model_description_md += f\" {one_model_md} |\"\n        if ct % 3 == 2:\n            model_description_md += \"\\n\"\n        ct += 1\n    return model_description_md\n\n\ndef build_about():\n    about_markdown = \"\"\"\n# About Us\nChatbot Arena is an open-source research project developed by members from [LMSYS](https://lmsys.org) and UC Berkeley [SkyLab](https://sky.cs.berkeley.edu/). Our mission is to build an open platform to evaluate LLMs by human preference in the real-world.\nWe open-source our [FastChat](https://github.com/lm-sys/FastChat) project at GitHub and release chat and human feedback dataset. We invite everyone to join us!\n\n## Arena Core Team\n- [Lianmin Zheng](https://lmzheng.net/) (co-lead), [Wei-Lin Chiang](https://infwinston.github.io/) (co-lead), [Ying Sheng](https://sites.google.com/view/yingsheng/home), [Joseph E. Gonzalez](https://people.eecs.berkeley.edu/~jegonzal/), [Ion Stoica](http://people.eecs.berkeley.edu/~istoica/)\n\n## Past Members\n- [Siyuan Zhuang](https://scholar.google.com/citations?user=KSZmI5EAAAAJ), [Hao Zhang](https://cseweb.ucsd.edu/~haozhang/)\n\n## Learn more\n- Chatbot Arena [paper](https://arxiv.org/abs/2403.04132), [launch blog](https://lmsys.org/blog/2023-05-03-arena/), [dataset](https://github.com/lm-sys/FastChat/blob/main/docs/dataset_release.md), [policy](https://lmsys.org/blog/2024-03-01-policy/)\n- LMSYS-Chat-1M dataset [paper](https://arxiv.org/abs/2309.11998), LLM Judge [paper](https://arxiv.org/abs/2306.05685)\n\n## Contact Us\n- Follow our [X](https://x.com/lmsysorg), [Discord](https://discord.gg/HSWAKCrnFx) or email us at lmsys.org@gmail.com\n- File issues on [GitHub](https://github.com/lm-sys/FastChat)\n- Download our datasets and models on [HuggingFace](https://huggingface.co/lmsys)\n\n## Acknowledgment\nWe thank [SkyPilot](https://github.com/skypilot-org/skypilot) and [Gradio](https://github.com/gradio-app/gradio) team for their system support.\nWe also thank [UC Berkeley SkyLab](https://sky.cs.berkeley.edu/), [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Hyperbolic](https://hyperbolic.xyz/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous sponsorship. Learn more about partnership [here](https://lmsys.org/donations/).\n\n<div class=\"sponsor-image-about\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/skylab.png\" alt=\"SkyLab\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/kaggle.png\" alt=\"Kaggle\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/mbzuai.jpeg\" alt=\"MBZUAI\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/a16z.jpeg\" alt=\"a16z\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/together.png\" alt=\"Together AI\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/hyperbolic_logo.png\" alt=\"Hyperbolic\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/anyscale.png\" alt=\"AnyScale\">\n    <img src=\"https://storage.googleapis.com/public-arena-asset/huggingface.png\" alt=\"HuggingFace\">\n</div>\n\"\"\"\n    gr.Markdown(about_markdown, elem_id=\"about_markdown\")\n\n\ndef build_single_model_ui(models, add_promotion_links=False):\n    promotion = (\n        \"\"\"\n- | [GitHub](https://github.com/lm-sys/FastChat) | [Dataset](https://github.com/lm-sys/FastChat/blob/main/docs/dataset_release.md) | [Twitter](https://twitter.com/lmsysorg) | [Discord](https://discord.gg/HSWAKCrnFx) |\n- Introducing Llama 2: The Next Generation Open Source Large Language Model. [[Website]](https://ai.meta.com/llama/)\n- Vicuna: An Open-Source Chatbot Impressing GPT-4 with 90% ChatGPT Quality. [[Blog]](https://lmsys.org/blog/2023-03-30-vicuna/)\n\n## 🤖 Choose any model to chat\n\"\"\"\n        if add_promotion_links\n        else \"\"\n    )\n\n    notice_markdown = f\"\"\"\n# 🏔️ Chat with Open Large Language Models\n{promotion}\n\"\"\"\n\n    state = gr.State()\n    gr.Markdown(notice_markdown, elem_id=\"notice_markdown\")\n\n    with gr.Group(elem_id=\"share-region-named\"):\n        with gr.Row(elem_id=\"model_selector_row\"):\n            model_selector = gr.Dropdown(\n                choices=models,\n                value=models[0] if len(models) > 0 else \"\",\n                interactive=True,\n                show_label=False,\n                container=False,\n            )\n        with gr.Row():\n            with gr.Accordion(\n                f\"🔍 Expand to see the descriptions of {len(models)} models\",\n                open=False,\n            ):\n                model_description_md = get_model_description_md(models)\n                gr.Markdown(model_description_md, elem_id=\"model_description_markdown\")\n\n        chatbot = gr.Chatbot(\n            elem_id=\"chatbot\",\n            label=\"Scroll down and start chatting\",\n            height=550,\n            show_copy_button=True,\n        )\n    with gr.Row():\n        textbox = gr.Textbox(\n            show_label=False,\n            placeholder=\"👉 Enter your prompt and press ENTER\",\n            elem_id=\"input_box\",\n        )\n        send_btn = gr.Button(value=\"Send\", variant=\"primary\", scale=0)\n\n    with gr.Row() as button_row:\n        upvote_btn = gr.Button(value=\"👍  Upvote\", interactive=False)\n        downvote_btn = gr.Button(value=\"👎  Downvote\", interactive=False)\n        flag_btn = gr.Button(value=\"⚠️  Flag\", interactive=False)\n        regenerate_btn = gr.Button(value=\"🔄  Regenerate\", interactive=False)\n        clear_btn = gr.Button(value=\"🗑️  Clear history\", interactive=False)\n\n    with gr.Accordion(\"Parameters\", open=False) as parameter_row:\n        temperature = gr.Slider(\n            minimum=0.0,\n            maximum=1.0,\n            value=0.7,\n            step=0.1,\n            interactive=True,\n            label=\"Temperature\",\n        )\n        top_p = gr.Slider(\n            minimum=0.0,\n            maximum=1.0,\n            value=1.0,\n            step=0.1,\n            interactive=True,\n            label=\"Top P\",\n        )\n        max_output_tokens = gr.Slider(\n            minimum=16,\n            maximum=2048,\n            value=1024,\n            step=64,\n            interactive=True,\n            label=\"Max output tokens\",\n        )\n\n    if add_promotion_links:\n        gr.Markdown(acknowledgment_md, elem_id=\"ack_markdown\")\n\n    # Register listeners\n    imagebox = gr.State(None)\n    btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]\n    upvote_btn.click(\n        upvote_last_response,\n        [state, model_selector],\n        [textbox, upvote_btn, downvote_btn, flag_btn],\n    )\n    downvote_btn.click(\n        downvote_last_response,\n        [state, model_selector],\n        [textbox, upvote_btn, downvote_btn, flag_btn],\n    )\n    flag_btn.click(\n        flag_last_response,\n        [state, model_selector],\n        [textbox, upvote_btn, downvote_btn, flag_btn],\n    )\n    regenerate_btn.click(\n        regenerate, state, [state, chatbot, textbox, imagebox] + btn_list\n    ).then(\n        bot_response,\n        [state, temperature, top_p, max_output_tokens],\n        [state, chatbot] + btn_list,\n    )\n    clear_btn.click(clear_history, None, [state, chatbot, textbox, imagebox] + btn_list)\n\n    model_selector.change(\n        clear_history, None, [state, chatbot, textbox, imagebox] + btn_list\n    )\n\n    textbox.submit(\n        add_text,\n        [state, model_selector, textbox, imagebox],\n        [state, chatbot, textbox, imagebox] + btn_list,\n    ).then(\n        bot_response,\n        [state, temperature, top_p, max_output_tokens],\n        [state, chatbot] + btn_list,\n    )\n    send_btn.click(\n        add_text,\n        [state, model_selector, textbox, imagebox],\n        [state, chatbot, textbox, imagebox] + btn_list,\n    ).then(\n        bot_response,\n        [state, temperature, top_p, max_output_tokens],\n        [state, chatbot] + btn_list,\n    )\n\n    return [state, model_selector]\n\n\ndef build_demo(models):\n    with gr.Blocks(\n        title=\"Chat with Open Large Language Models\",\n        theme=gr.themes.Default(),\n        css=block_css,\n    ) as demo:\n        url_params = gr.JSON(visible=False)\n\n        state, model_selector = build_single_model_ui(models)\n\n        if args.model_list_mode not in [\"once\", \"reload\"]:\n            raise ValueError(f\"Unknown model list mode: {args.model_list_mode}\")\n\n        if args.show_terms_of_use:\n            load_js = get_window_url_params_with_tos_js\n        else:\n            load_js = get_window_url_params_js\n\n        demo.load(\n            load_demo,\n            [url_params],\n            [\n                state,\n                model_selector,\n            ],\n            js=load_js,\n        )\n\n    return demo\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--host\", type=str, default=\"0.0.0.0\")\n    parser.add_argument(\"--port\", type=int)\n    parser.add_argument(\n        \"--share\",\n        action=\"store_true\",\n        help=\"Whether to generate a public, shareable link\",\n    )\n    parser.add_argument(\n        \"--controller-url\",\n        type=str,\n        default=\"http://localhost:21001\",\n        help=\"The address of the controller\",\n    )\n    parser.add_argument(\n        \"--concurrency-count\",\n        type=int,\n        default=10,\n        help=\"The concurrency count of the gradio queue\",\n    )\n    parser.add_argument(\n        \"--model-list-mode\",\n        type=str,\n        default=\"once\",\n        choices=[\"once\", \"reload\"],\n        help=\"Whether to load the model list once or reload the model list every time\",\n    )\n    parser.add_argument(\n        \"--moderate\",\n        action=\"store_true\",\n        help=\"Enable content moderation to block unsafe inputs\",\n    )\n    parser.add_argument(\n        \"--show-terms-of-use\",\n        action=\"store_true\",\n        help=\"Shows term of use before loading the demo\",\n    )\n    parser.add_argument(\n        \"--register-api-endpoint-file\",\n        type=str,\n        help=\"Register API-based model endpoints from a JSON file\",\n    )\n    parser.add_argument(\n        \"--gradio-auth-path\",\n        type=str,\n        help='Set the gradio authentication file path. The file should contain one or more user:password pairs in this format: \"u1:p1,u2:p2,u3:p3\"',\n    )\n    parser.add_argument(\n        \"--gradio-root-path\",\n        type=str,\n        help=\"Sets the gradio root path, eg /abc/def. Useful when running behind a reverse-proxy or at a custom URL path prefix\",\n    )\n    parser.add_argument(\n        \"--use-remote-storage\",\n        action=\"store_true\",\n        default=False,\n        help=\"Uploads image files to google cloud storage if set to true\",\n    )\n    args = parser.parse_args()\n    logger.info(f\"args: {args}\")\n\n    # Set global variables\n    set_global_vars(args.controller_url, args.moderate, args.use_remote_storage)\n    models, all_models = get_model_list(\n        args.controller_url, args.register_api_endpoint_file, vision_arena=False\n    )\n\n    # Set authorization credentials\n    auth = None\n    if args.gradio_auth_path is not None:\n        auth = parse_gradio_auth_creds(args.gradio_auth_path)\n\n    # Launch the demo\n    demo = build_demo(models)\n    demo.queue(\n        default_concurrency_limit=args.concurrency_count,\n        status_update_rate=10,\n        api_open=False,\n    ).launch(\n        server_name=args.host,\n        server_port=args.port,\n        share=args.share,\n        max_threads=200,\n        auth=auth,\n        root_path=args.gradio_root_path,\n    )\n"}}}}