doc/colab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Welcome to the ExlV2 webui Colab Notebook\n",
    "It's really easy to get started. Just press the two **Play** buttons below, and then connect to the **Cloudflare URL** shown at the end.\n",
    "\n",
    "\n",
    "**Keep this page open and occationally check for captcha's so that your AI is not shut down**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/turboderp/exui\n",
    "!cd exui && pip install -r requirements.txt\n",
    "!git clone https://github.com/turboderp/exllamav2\n",
    "!cd exllamav2 && python setup.py install --user "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "Method = \"EXL2\" # @param [\"EXL2\", \"GptQ\"] {allow-input: true}\n",
    "Model = \"turboderp/Llama2-13B-exl2\" #@param [\"\"]{allow-input: true}\n",
    "bpw = \"2.5bpw\" # @param [\"2.5bpw\", \"4.0bpw\", \"8.0bpw\"] {allow-input: true}\n",
    "!pip install -U huggingface-hub\n",
    "!mkdir models\n",
    "if Method:=\"EXL2\":\n",
    "  !cd models && huggingface-cli download $Model --revision {bpw} --local-dir /content/models --local-dir-use-symlinks False\n",
    "else:\n",
    "  !cd models && huggingface-cli download $Model --local-dir /content/models --local-dir-use-symlinks False\n",
    "\n",
    "  \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "!wget -c https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64\n",
    "!chmod +x cloudflared-linux-amd64\n",
    "!nohup ./cloudflared-linux-amd64 tunnel --url http://localhost:5000 &\n",
    "!sleep 5\n",
    "!cat nohup.out\n",
    "!cd exui && python server.py --host 127.0.0.1:5000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}