dependencies.yaml

# Dependency list for https://github.com/rapidsai/dependency-file-generator
files:
  all:
    output: conda
    matrix:
      cuda: ["11.8"]
      arch: [x86_64]
    includes:
      - build_all
      - build_cpp
      - build_wheels
      - build_python
      - build_python_common
      - cudatoolkit
      - develop
      - docs
      - notebooks
      - py_version
      - run_common
      - run_cudf
      - run_dask_cudf
      - run_cudf_kafka
      - run_custreamz
      - test_cpp
      - test_python_common
      - test_python_cudf
      - test_python_dask_cudf
  test_cpp:
    output: none
    includes:
      - cudatoolkit
      - test_cpp
      - libarrow_run
  test_python:
    output: none
    includes:
      - cudatoolkit
      - py_version
      - test_python_common
      - test_python_cudf
      - test_python_dask_cudf
      - pyarrow_run
  test_java:
    output: none
    includes:
      - build_all
      - libarrow_run
      - cudatoolkit
      - test_java
  test_notebooks:
    output: none
    includes:
      - notebooks
      - py_version
  checks:
    output: none
    includes:
      - develop
      - py_version
  docs:
    output: none
    includes:
      - cudatoolkit
      - docs
      - py_version
  py_build_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: build-system
    includes:
      - build_all
      - build_python
      - build_python_common
      - build_wheels
  py_run_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: project
    includes:
      - run_common
      - run_cudf
      - pyarrow_run
  py_test_cudf:
    output: pyproject
    pyproject_dir: python/cudf
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
      - test_python_cudf
  py_build_dask_cudf:
    output: pyproject
    pyproject_dir: python/dask_cudf
    extras:
      table: build-system
    includes:
      - build_wheels
  py_run_dask_cudf:
    output: pyproject
    pyproject_dir: python/dask_cudf
    extras:
      table: project
    includes:
      - run_common
      - run_dask_cudf
  py_test_dask_cudf:
    output: pyproject
    pyproject_dir: python/dask_cudf
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
      - test_python_dask_cudf
  py_build_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: build-system
    includes:
      - build_wheels
      - build_python_common
  py_run_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: project
    includes:
      - run_cudf_kafka
  py_test_cudf_kafka:
    output: pyproject
    pyproject_dir: python/cudf_kafka
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
  py_build_custreamz:
    output: pyproject
    pyproject_dir: python/custreamz
    extras:
      table: build-system
    includes:
      - build_wheels
  py_run_custreamz:
    output: pyproject
    pyproject_dir: python/custreamz
    extras:
      table: project
    includes:
      - run_custreamz
  py_test_custreamz:
    output: pyproject
    pyproject_dir: python/custreamz
    extras:
      table: project.optional-dependencies
      key: test
    includes:
      - test_python_common
channels:
  - rapidsai
  - rapidsai-nightly
  - dask/label/dev
  - pytorch
  - conda-forge
  - nvidia
dependencies:
  build_all:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - &cmake_ver cmake>=3.23.1,!=3.25.0
          - ninja
      - output_types: conda
        packages:
          - c-compiler
          - cxx-compiler
          - dlpack>=0.5,<0.6.0a0
    specific:
      - output_types: conda
        matrices:
          - matrix:
              arch: x86_64
            packages:
              - gcc_linux-64=11.*
              - sysroot_linux-64==2.17
          - matrix:
              arch: aarch64
            packages:
              - gcc_linux-aarch64=11.*
              - sysroot_linux-aarch64==2.17
      - output_types: conda
        matrices:
          - matrix:
              arch: x86_64
              cuda: "11.8"
            packages:
              - nvcc_linux-64=11.8
          - matrix:
              arch: aarch64
              cuda: "11.8"
            packages:
              - nvcc_linux-aarch64=11.8
  build_cpp:
    common:
      - output_types: [conda, requirements]
        packages:
          - librmm==23.4.*
      - output_types: conda
        packages:
          - fmt>=9.1.0,<10
          - &gtest gtest==1.10.0.*
          - &gmock gmock==1.10.0.*
          # Hard pin the patch version used during the build. This must be kept
          # in sync with the version pinned in get_arrow.cmake.
          - libarrow==10.0.1.*
          - librdkafka=1.7.0
          - spdlog>=1.11.0,<1.12
  build_wheels:
    common:
      - output_types: pyproject
        packages:
          - wheel
          - setuptools
  build_python_common:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - cython>=0.29,<0.30
          # Hard pin the patch version used during the build. This must be kept
          # in sync with the version pinned in get_arrow.cmake.
          - pyarrow==10.0.1.*
          - numpy>=1.21
  build_python:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - scikit-build>=0.13.1,<0.17.2
          - rmm==23.4.*
      - output_types: conda
        packages:
          - &protobuf protobuf>=4.21.6,<4.22
      - output_types: pyproject
        packages:
          - protoc-wheel
  libarrow_run:
    common:
      - output_types: [conda, requirements]
        packages:
          # Allow runtime version to float up to minor version
          - libarrow==10.*
  pyarrow_run:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          # Allow runtime version to float up to minor version
          - pyarrow==10.*
  cudatoolkit:
    specific:
      - output_types: conda
        matrices:
          - matrix:
              cuda: "11.8"
            packages:
              - cudatoolkit=11.8
              - libcurand-dev=10.3.0.86
              - libcurand=10.3.0.86
          - matrix:
              cuda: "11.5"
            packages:
              - cudatoolkit=11.5
                # Can't hard pin the version since 11.x is missing many
                # packages for specific versions
              - libcurand-dev>=10.2.6.48,<=10.2.7.107
              - libcurand>=10.2.6.48,<=10.2.7.107
          - matrix:
              cuda: "11.4"
            packages:
              - cudatoolkit=11.4
              - &libcurand_dev114 libcurand-dev>=10.2.5.43,<=10.2.5.120
              - &libcurand114 libcurand>=10.2.5.43,<=10.2.5.120
          - matrix:
              cuda: "11.2"
            packages:
              - cudatoolkit=11.2
                # The NVIDIA channel doesn't publish pkgs older than 11.4 for
                # these libs, so 11.2 uses 11.4 packages (the oldest
                # available).
              - *libcurand_dev114
              - *libcurand114
  develop:
    common:
      - output_types: [conda, requirements]
        packages:
          - pre-commit
      - output_types: conda
        packages:
          - doxygen=1.8.20 # pre-commit hook needs a specific version.
  docs:
    common:
      - output_types: [conda]
        packages:
          - doxygen=1.8.20
          - myst-nb
          - nbsphinx
          - numpydoc
          - pandoc
          - pydata-sphinx-theme
          - sphinx
          - sphinx-autobuild
          - sphinx-copybutton
          - sphinx-markdown-tables
          - sphinxcontrib-websupport
  notebooks:
    common:
      - output_types: [conda, requirements]
        packages:
          - ipython
          - notebook
          - scipy
  py_version:
    specific:
      - output_types: conda
        matrices:
          - matrix:
              py: "3.8"
            packages:
              - python=3.8
          - matrix:
              py: "3.9"
            packages:
              - python=3.9
          - matrix:
              py: "3.10"
            packages:
              - python=3.10
          - matrix:
            packages:
              - python>=3.8,<3.11
  run_common:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - fsspec>=0.6.0
          - numpy>=1.21
          - pandas>=1.3,<1.6.0dev0
  run_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - cachetools
          - cuda-python>=11.7.1,<12.0
          - &numba numba>=0.56.4,<0.57
          - nvtx>=0.2.1
          - packaging
          - rmm==23.4.*
          - typing_extensions
          - *protobuf
      - output_types: conda
        packages:
          - cubinlinker
          - cupy>=9.5.0,<12.0.0a0
          - pip
          - pip:
              - git+https://github.com/python-streamz/streamz.git@master
          - ptxcompiler
      - output_types: requirements
        packages:
          # pip recognizes the index as a global option for the requirements.txt file
          # This index is needed for rmm, cubinlinker, ptxcompiler.
          - --extra-index-url=https://pypi.ngc.nvidia.com
          - cubinlinker-cu11
          - git+https://github.com/python-streamz/streamz.git@master
          - ptxcompiler-cu11
      - output_types: pyproject
        packages:
          - cubinlinker
          - &cupy_pip cupy-cuda11x>=9.5.0,<12.0.0a0
          - ptxcompiler
    specific:
      - output_types: requirements
        matrices:
          - matrix:
              arch: x86_64
            packages:
              - cupy-cuda115>=9.5.0,<12.0.0a0 # TODO: This might change to cupy-cuda11x?
          - matrix:
              arch: aarch64
            packages:
              - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works.
  run_dask_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - dask==2023.3.2
          - distributed==2023.3.2.1
      - output_types: conda
        packages:
          - dask-core==2023.3.2  # dask-core in conda is the actual package & dask is the meta package
      - output_types: pyproject
        packages:
          - &cudf cudf==23.4.*
          - *cupy_pip
  run_cudf_kafka:
    common:
      - output_types: conda
        packages:
          - python-confluent-kafka==1.7.0
      - output_types: [requirements, pyproject]
        packages:
          - *cudf
          - confluent-kafka==1.7.0
  run_custreamz:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - streamz
      - output_types: [requirements, pyproject]
        packages:
          - *cudf
          - cudf_kafka==23.4.*
  test_cpp:
    common:
      - output_types: conda
        packages:
          - *cmake_ver
          - *gtest
          - *gmock
    specific:
      - output_types: conda
        matrices:
          - matrix:
              cuda: "11.8"
            packages:
              - cuda-sanitizer-api=11.8.86
          - matrix:
            packages:
  test_java:
    common:
      - output_types: conda
        packages:
          - *cmake_ver
          - maven
          - openjdk=8.*
    specific:
      - output_types: conda
        matrices:
          - matrix:
              cuda: "11.5"
            packages:
              - cuda-nvtx=11.5
          - matrix:
              cuda: "11.8"
            packages:
              - cuda-nvtx=11.8
  test_python_common:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - pytest
          - pytest-cov
          - pytest-xdist
  test_python_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - fastavro>=0.22.9
          - hypothesis
          - mimesis>=4.1.0
          - pyorc
          - pytest-benchmark
          - pytest-cases
          - python-snappy>=0.6.0
          - scipy
      - output_types: conda
        packages:
          - aiobotocore>=2.2.0
          - boto3>=1.21.21
          - botocore>=1.24.21
          - msgpack-python
          - moto>=4.0.8
          - s3fs>=2022.3.0
      - output_types: pyproject
        packages:
          - msgpack
          - &tokenizers tokenizers==0.13.1
          - &transformers transformers==4.24.0
          - tzdata
    specific:
      - output_types: conda
        matrices:
          - matrix:
              arch: x86_64
            packages:
              # Currently, CUDA builds of pytorch do not exist for aarch64. We require
              # version <1.12.0 because newer versions use nvidia::cuda-toolkit.
              - pytorch<1.12.0
              # We only install these on x86_64 to avoid pulling pytorch as a
              # dependency of transformers.
              - *tokenizers
              - *transformers
          - matrix:
            packages:
  test_python_dask_cudf:
    common:
      - output_types: [conda, requirements, pyproject]
        packages:
          - dask-cuda==23.4.*
          - *numba