examples/demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5gyFPhPWJev9"
      },
      "source": [
        "##### Copyright 2019 DeepMind Technologies Limited."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "LPGlYwKdJP3o"
      },
      "outputs": [],
      "source": [
        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "czUOoEyaLw4_"
      },
      "source": [
        "# Environments\n",
        "\n",
        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
        "  \u003ctd\u003e\n",
        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/deepmind/reverb/blob/master/examples/demo.ipynb\"\u003e\n",
        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n",
        "    Run in Google Colab\u003c/a\u003e\n",
        "  \u003c/td\u003e\n",
        "  \u003ctd\u003e\n",
        "    \u003ca target=\"_blank\" href=\"https://github.com/deepmind/reverb/blob/master/examples/demo.ipynb\"\u003e\n",
        "    \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n",
        "    View source on GitHub\u003c/a\u003e\n",
        "  \u003c/td\u003e\n",
        "\u003c/table\u003e"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ISYJFHZ1Lh8P"
      },
      "source": [
        "# Introduction\n",
        "\n",
        "This colab is a demonstration of how to use Reverb through examples."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vQA3Gy4cKX05"
      },
      "source": [
        "# Setup\n",
        "\n",
        "Installs the stable build of Reverb (dm-reverb) and TensorFlow (tf) to match."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "skip": true
          }
        },
        "id": "GrkMkCjA7_tK"
      },
      "outputs": [],
      "source": [
        "!pip install numpy --upgrade\n",
        "!pip install dm-tree\n",
        "!pip install dm-reverb[tensorflow]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "NSzJMNvXuPi9"
      },
      "outputs": [],
      "source": [
        "import reverb\n",
        "import tensorflow as tf"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zD_GkERHMf-E"
      },
      "source": [
        "The code below defines a dummy RL environment for use in the examples below."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "uEYL3Gn3MLbS"
      },
      "outputs": [],
      "source": [
        "OBSERVATION_SPEC = tf.TensorSpec([10, 10], tf.uint8)\n",
        "ACTION_SPEC = tf.TensorSpec([2], tf.float32)\n",
        "\n",
        "def agent_step(unused_timestep) -\u003e tf.Tensor:\n",
        "  return tf.cast(tf.random.uniform(ACTION_SPEC.shape) \u003e .5,\n",
        "                 ACTION_SPEC.dtype)\n",
        "\n",
        "def environment_step(unused_action) -\u003e tf.Tensor:\n",
        "  return tf.cast(tf.random.uniform(OBSERVATION_SPEC.shape, maxval=256),\n",
        "                 OBSERVATION_SPEC.dtype)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cESH6nbOCRjS"
      },
      "source": [
        "# Creating a Server and Client"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "Zm5uTw1fvknY"
      },
      "outputs": [],
      "source": [
        "# Initialize the reverb server.\n",
        "simple_server = reverb.Server(\n",
        "    tables=[\n",
        "        reverb.Table(\n",
        "            name='my_table',\n",
        "            sampler=reverb.selectors.Prioritized(priority_exponent=0.8),\n",
        "            remover=reverb.selectors.Fifo(),\n",
        "            max_size=int(1e6),\n",
        "            # Sets Rate Limiter to a low number for the examples.\n",
        "            # Read the Rate Limiters section for usage info.\n",
        "            rate_limiter=reverb.rate_limiters.MinSize(2),\n",
        "            # The signature is optional but it is good practice to set it as it\n",
        "            # enables data validation and easier dataset construction. Note that\n",
        "            # we prefix all shapes with a 3 as the trajectories we'll be writing\n",
        "            # consist of 3 timesteps.\n",
        "            signature={\n",
        "                'actions':\n",
        "                    tf.TensorSpec([3, *ACTION_SPEC.shape], ACTION_SPEC.dtype),\n",
        "                'observations':\n",
        "                    tf.TensorSpec([3, *OBSERVATION_SPEC.shape],\n",
        "                                  OBSERVATION_SPEC.dtype),\n",
        "            },\n",
        "        )\n",
        "    ],\n",
        "    # Sets the port to None to make the server pick one automatically.\n",
        "    # This can be omitted as it's the default.\n",
        "    port=None)\n",
        "\n",
        "# Initializes the reverb client on the same port as the server.\n",
        "client = reverb.Client(f'localhost:{simple_server.port}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fFyKp_toviQK"
      },
      "source": [
        "For details on customizing the sampler, remover, and rate limiter, see below."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5xERcgn8v0Li"
      },
      "source": [
        "# Example 1: Overlapping Trajectories\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lcs55Fnk3SBD"
      },
      "source": [
        "## Inserting Overlapping Trajectories"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "3xs-tSRWcI6x"
      },
      "outputs": [],
      "source": [
        "# Dynamically adds trajectories of length 3 to 'my_table' using a client writer.\n",
        "\n",
        "with client.trajectory_writer(num_keep_alive_refs=3) as writer:\n",
        "  timestep = environment_step(None)\n",
        "  for step in range(4):\n",
        "    action = agent_step(timestep)\n",
        "    writer.append({'action': action, 'observation': timestep})\n",
        "    timestep = environment_step(action)\n",
        "\n",
        "    if step \u003e= 2:\n",
        "      # In this example, the item consists of the 3 most recent timesteps that\n",
        "      # were added to the writer and has a priority of 1.5.\n",
        "      writer.create_item(\n",
        "          table='my_table',\n",
        "          priority=1.5,\n",
        "          trajectory={\n",
        "              'actions': writer.history['action'][-3:],\n",
        "              'observations': writer.history['observation'][-3:],\n",
        "          }\n",
        "      )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iohnQl20x3bz"
      },
      "source": [
        "The animation illustrates the state of the server at each step in the\n",
        "above code block. Although each item is being set to have the same\n",
        "priority value of 1.5, items do not need to have the same priority values.\n",
        "In real world scenarios, items would have differing and\n",
        "dynamically-calculated priority values.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hK90myIgxuGL"
      },
      "source": [
        "\u003cimg src=\"https://raw.githubusercontent.com/deepmind/reverb/master/docs/animations/diagram1.svg\" /\u003e"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4z0zD24bv5Ju"
      },
      "source": [
        "## Sampling Overlapping Trajectories in TensorFlow"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "SQFSZJkyroFX"
      },
      "outputs": [],
      "source": [
        "# Dataset samples sequences of length 3 and streams the timesteps one by one.\n",
        "# This allows streaming large sequences that do not necessarily fit in memory.\n",
        "dataset = reverb.TrajectoryDataset.from_table_signature(\n",
        "  server_address=f'localhost:{simple_server.port}',\n",
        "  table='my_table',\n",
        "  max_in_flight_samples_per_worker=10)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "UaICLHYqC3g7"
      },
      "outputs": [],
      "source": [
        "# Batches 2 sequences together.\n",
        "# Shapes of items is now [2, 3, 10, 10].\n",
        "batched_dataset = dataset.batch(2)\n",
        "\n",
        "for sample in batched_dataset.take(1):\n",
        "  # Results in the following format.\n",
        "  print(sample.info.key)              # ([2], uint64)\n",
        "  print(sample.info.probability)      # ([2], float64)\n",
        "\n",
        "  print(sample.data['observations'])  # ([2, 3, 10, 10], uint8)\n",
        "  print(sample.data['actions'])       # ([2, 3, 2], float32)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Xt0Kec6CU6F"
      },
      "source": [
        "# Example 2: Complete Episodes"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1S1aSGmJ35ql"
      },
      "source": [
        "Create a new server for this example to keep the elements of the priority table consistent."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "_WsESIry3xgd"
      },
      "outputs": [],
      "source": [
        "EPISODE_LENGTH = 150\n",
        "\n",
        "complete_episode_server = reverb.Server(tables=[\n",
        "    reverb.Table(\n",
        "        name='my_table',\n",
        "        sampler=reverb.selectors.Prioritized(priority_exponent=0.8),\n",
        "        remover=reverb.selectors.Fifo(),\n",
        "        max_size=int(1e6),\n",
        "        # Sets Rate Limiter to a low number for the examples.\n",
        "        # Read the Rate Limiters section for usage info.\n",
        "        rate_limiter=reverb.rate_limiters.MinSize(2),\n",
        "        # The signature is optional but it is good practice to set it as it\n",
        "        # enables data validation and easier dataset construction. Note that\n",
        "        # the number of observations is larger than the number of actions.\n",
        "        # The extra observation is the terminal state where no action is\n",
        "        # taken.\n",
        "        signature={\n",
        "            'actions':\n",
        "                tf.TensorSpec([EPISODE_LENGTH, *ACTION_SPEC.shape],\n",
        "                              ACTION_SPEC.dtype),\n",
        "            'observations':\n",
        "                tf.TensorSpec([EPISODE_LENGTH + 1, *OBSERVATION_SPEC.shape],\n",
        "                              OBSERVATION_SPEC.dtype),\n",
        "        },\n",
        "    ),\n",
        "])\n",
        "\n",
        "# Initializes the reverb client on the same port.\n",
        "client = reverb.Client(f'localhost:{complete_episode_server.port}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KBLN137F3x3H"
      },
      "source": [
        "## Inserting Complete Episodes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "kEstKtvu-0hi"
      },
      "outputs": [],
      "source": [
        "# Writes whole episodes of varying length to a Reverb server.\n",
        "\n",
        "NUM_EPISODES = 10\n",
        "\n",
        "# We know that episodes are at most 150 steps so we set the writer buffer size\n",
        "# to 151 (to capture the final observation).\n",
        "with client.trajectory_writer(num_keep_alive_refs=151) as writer:\n",
        "  for _ in range(NUM_EPISODES):\n",
        "    timestep = environment_step(None)\n",
        "\n",
        "    for _ in range(EPISODE_LENGTH):\n",
        "      action = agent_step(timestep)\n",
        "      writer.append({'action': action, 'observation': timestep})\n",
        "\n",
        "      timestep = environment_step(action)\n",
        "\n",
        "    # The astute reader will recognize that the final timestep has not been\n",
        "    # appended to the writer. We'll go ahead and add it WITHOUT an action. The\n",
        "    # writer will automatically fill in the gap with `None` for the action\n",
        "    # column.\n",
        "    writer.append({'observation': timestep})\n",
        "\n",
        "    # Now that the entire episode has been added to the writer buffer we can an\n",
        "    # item with a trajectory that spans the entire episode. Note that the final\n",
        "    # action must not be included as it is None and the trajectory would be\n",
        "    # rejected if we tried to include it.\n",
        "    writer.create_item(\n",
        "        table='my_table',\n",
        "        priority=1.5,\n",
        "        trajectory={\n",
        "            'actions': writer.history['action'][:-1],\n",
        "            'observations': writer.history['observation'][:],\n",
        "        })\n",
        "\n",
        "    # This call blocks until all the items (in this case only one) have been\n",
        "    # sent to the server, inserted into respective tables and confirmations\n",
        "    # received by the writer.\n",
        "    writer.end_episode(timeout_ms=1000)\n",
        "\n",
        "    # Ending the episode also clears the history property which is why we are\n",
        "    # able to use `[:]` in when defining the trajectory above.\n",
        "    assert len(writer.history['action']) == 0\n",
        "    assert len(writer.history['observation']) == 0"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0IRuThqjwg8G"
      },
      "source": [
        "## Sampling Complete Episodes in TensorFlow"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "y4_gzMHN-yc5"
      },
      "outputs": [],
      "source": [
        "# Each sample is an entire episode.\n",
        "# Adjusts the expected shapes to account for the whole episode length.\n",
        "dataset = reverb.TrajectoryDataset.from_table_signature(\n",
        "  server_address=f'localhost:{complete_episode_server.port}',\n",
        "  table='my_table',\n",
        "  max_in_flight_samples_per_worker=10,\n",
        "  rate_limiter_timeout_ms=10)\n",
        "\n",
        "# Batches 128 episodes together.\n",
        "# Each item is an episode of the format (observations, actions) as above.\n",
        "# Shape of items are now ([128, 151, 10, 10], [128, 150, 2]).\n",
        "dataset = dataset.batch(128)\n",
        "\n",
        "# Sample has type reverb.ReplaySample.\n",
        "for sample in dataset.take(1):\n",
        "  # Results in the following format.\n",
        "  print(sample.info.key)              # ([128], uint64)\n",
        "  print(sample.info.probability)      # ([128], float64)\n",
        "\n",
        "  print(sample.data['observations'])  # ([128, 151, 10, 10], uint8)\n",
        "  print(sample.data['actions'])       # ([128, 150, 2], float32)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UD9u5-i6A9ub"
      },
      "source": [
        "# Example 3: Multiple Priority Tables"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "54VsdWyg4VGi"
      },
      "source": [
        "Create a server that maintains multiple priority tables."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "Co0i6GCAG4Xc"
      },
      "outputs": [],
      "source": [
        "multitable_server = reverb.Server(\n",
        "    tables=[\n",
        "        reverb.Table(\n",
        "            name='my_table_a',\n",
        "            sampler=reverb.selectors.Prioritized(priority_exponent=0.8),\n",
        "            remover=reverb.selectors.Fifo(),\n",
        "            max_size=int(1e6),\n",
        "            # Sets Rate Limiter to a low number for the examples.\n",
        "            # Read the Rate Limiters section for usage info.\n",
        "            rate_limiter=reverb.rate_limiters.MinSize(1)),\n",
        "        reverb.Table(\n",
        "            name='my_table_b',\n",
        "            sampler=reverb.selectors.Prioritized(priority_exponent=0.8),\n",
        "            remover=reverb.selectors.Fifo(),\n",
        "            max_size=int(1e6),\n",
        "            # Sets Rate Limiter to a low number for the examples.\n",
        "            # Read the Rate Limiters section for usage info.\n",
        "            rate_limiter=reverb.rate_limiters.MinSize(1)),\n",
        "    ])\n",
        "\n",
        "client = reverb.Client('localhost:{}'.format(multitable_server.port))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AaufIpBZxHv1"
      },
      "source": [
        "## Inserting Sequences of Varying Length into Multiple Priority Tables\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "YoSLa6QvxGxq"
      },
      "outputs": [],
      "source": [
        "with client.trajectory_writer(num_keep_alive_refs=3) as writer:\n",
        "  timestep = environment_step(None)\n",
        "\n",
        "  for step in range(4):\n",
        "    writer.append({'timestep': timestep})\n",
        "\n",
        "    action = agent_step(timestep)\n",
        "    timestep = environment_step(action)\n",
        "\n",
        "    if step \u003e= 1:\n",
        "      writer.create_item(\n",
        "          table='my_table_b',\n",
        "          priority=4-step,\n",
        "          trajectory=writer.history['timestep'][-2:])\n",
        "\n",
        "    if step \u003e= 2:\n",
        "      writer.create_item(\n",
        "          table='my_table_a',\n",
        "          priority=4-step,\n",
        "          trajectory=writer.history['timestep'][-3:])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b8ffbswoxYqE"
      },
      "source": [
        "This diagram shows the state of the server after executing the above cell.\n",
        "\n",
        "\n",
        "\u003cimg src=\"https://raw.githubusercontent.com/deepmind/reverb/master/docs/animations/diagram2.svg\" /\u003e"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2aQn_QAsIgmZ"
      },
      "source": [
        "# Example 4: Samplers and Removers\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oh3pY0C6AuNA"
      },
      "source": [
        "##  Creating a Server with a Prioritized Sampler and a FIFO Remover"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "ifjWSrIiIlGK"
      },
      "outputs": [],
      "source": [
        "reverb.Server(tables=[\n",
        "    reverb.Table(\n",
        "        name='my_table',\n",
        "        sampler=reverb.selectors.Prioritized(priority_exponent=0.8),\n",
        "        remover=reverb.selectors.Fifo(),\n",
        "        max_size=int(1e6),\n",
        "        rate_limiter=reverb.rate_limiters.MinSize(100)),\n",
        "])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KAJf-1UJAzgR"
      },
      "source": [
        "## Creating a Server with a MaxHeap Sampler and a MinHeap Remover"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y9oldFAdP0j6"
      },
      "source": [
        "Setting `max_times_sampled=1` causes each item to be removed after it is\n",
        "sampled once. The end result is a priority table that essentially functions\n",
        "as a max priority queue.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "ZYkeRJ3mAyy4"
      },
      "outputs": [],
      "source": [
        "max_size = 1000\n",
        "reverb.Server(tables=[\n",
        "    reverb.Table(\n",
        "        name='my_priority_queue',\n",
        "        sampler=reverb.selectors.MaxHeap(),\n",
        "        remover=reverb.selectors.MinHeap(),\n",
        "        max_size=max_size,\n",
        "        rate_limiter=reverb.rate_limiters.MinSize(int(0.95 * max_size)),\n",
        "        max_times_sampled=1,\n",
        "    )\n",
        "])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lRLLMHR2A6oN"
      },
      "source": [
        "## Creating a Server with One Queue and One Circular Buffer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EXma2uTiPkOy"
      },
      "source": [
        "Behavior of canonical data structures such as\n",
        "[circular buffer](https://en.wikipedia.org/wiki/Circular_buffer) or a max\n",
        "[priority queue](https://en.wikipedia.org/wiki/Priority_queue) can\n",
        "be implemented in Reverb by modifying the `sampler` and `remover`\n",
        "or by using the `PriorityTable` queue initializer."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "mH3MmDJSIoog"
      },
      "outputs": [],
      "source": [
        "reverb.Server(\n",
        "    tables=[\n",
        "        reverb.Table.queue(name='my_queue', max_size=10000),\n",
        "        reverb.Table(\n",
        "            name='my_circular_buffer',\n",
        "            sampler=reverb.selectors.Fifo(),\n",
        "            remover=reverb.selectors.Fifo(),\n",
        "            max_size=10000,\n",
        "            max_times_sampled=1,\n",
        "            rate_limiter=reverb.rate_limiters.MinSize(1)),\n",
        "    ])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9NkojFD_BURQ"
      },
      "source": [
        "# Example 5: Rate Limiters\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VYGYIpld5Fmu"
      },
      "source": [
        "## Creating a Server with a SampleToInsertRatio Rate Limiter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "test": {
            "output": "ignore",
            "timeout": 300
          }
        },
        "id": "LwpVpjfxBWRA"
      },
      "outputs": [],
      "source": [
        "reverb.Server(\n",
        "    tables=[\n",
        "        reverb.Table(\n",
        "            name='my_table',\n",
        "            sampler=reverb.selectors.Prioritized(priority_exponent=0.8),\n",
        "            remover=reverb.selectors.Fifo(),\n",
        "            max_size=int(1e6),\n",
        "            rate_limiter=reverb.rate_limiters.SampleToInsertRatio(\n",
        "                samples_per_insert=3.0, min_size_to_sample=3,\n",
        "                error_buffer=3.0)),\n",
        "    ])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p98u_jg3CAxG"
      },
      "source": [
        "\n",
        "This example is intended to be used in a distributed or multi-threaded\n",
        "enviroment where insertion blocking will be unblocked by sample calls from\n",
        "an independent thread. If the system is single threaded, the blocked\n",
        "insertion call will cause a deadlock.\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "demo.ipynb",
      "private_outputs": true,
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}