Remove torchdata dependency from package and from CI (#2241)

pytorch · Mar 22, 2024 · 51f373d · 51f373d
1 parent 52c0d85
commit 51f373d
Show file tree

Hide file tree

Showing 24 changed files with 15 additions and 120 deletions.
diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 unset PYTORCH_VERSION
-unset TORCHDATA_VERSION
 # For unittest, nightly PyTorch is used as the following section,
 # so no need to set PYTORCH_VERSION.
 # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
@@ -30,10 +29,6 @@ printf "* Installing PyTorch\n"
 )
 
 
-printf "Installing torchdata nightly with portalocker\n"
-pip install "portalocker>=2.0.0"
-pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu
-
 printf "* Installing torchtext\n"
 python setup.py develop
 

diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 unset PYTORCH_VERSION
-unset TORCHDATA_VERSION
 # For unittest, nightly PyTorch is used as the following section,
 # so no need to set PYTORCH_VERSION.
 # In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
@@ -19,10 +18,6 @@ conda activate ./env
 printf "* Installing PyTorch\n"
 conda install -y -c "pytorch-${UPLOAD_CHANNEL}" ${CONDA_CHANNEL_FLAGS} pytorch cpuonly
 
-printf "* Installing torchdata nightly with portalocker\n"
-pip install "portalocker>=2.0.0"
-pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu
-
 printf "* Installing pywin32_postinstall script\n"
 curl --output pywin32_postinstall.py https://raw.githubusercontent.com/mhammond/pywin32/main/pywin32_postinstall.py
 python pywin32_postinstall.py -install

diff --git a/.github/workflows/build-conda-linux.yml b/.github/workflows/build-conda-linux.yml
@@ -29,7 +29,7 @@ jobs:
       matrix:
         include:
           - repository: pytorch/text
-            pre-script: packaging/install_torchdata.sh
+            pre-script: ""
             post-script: ""
             conda-package-directory: packaging/torchtext
             smoke-test-script: test/smoke_tests/smoke_tests.py

diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml
@@ -28,7 +28,7 @@ jobs:
       matrix:
         include:
           - repository: pytorch/text
-            pre-script: packaging/install_torchdata.sh
+            pre-script: ""
             post-script: ""
             conda-package-directory: packaging/torchtext
             smoke-test-script: test/smoke_tests/smoke_tests.py

diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml
@@ -29,7 +29,7 @@ jobs:
       matrix:
         include:
           - repository: pytorch/text
-            pre-script: packaging/install_torchdata.sh
+            pre-script: ""
             post-script: ""
             conda-package-directory: packaging/torchtext
             smoke-test-script: test/smoke_tests/smoke_tests.py

diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml
@@ -34,7 +34,7 @@ jobs:
       matrix:
         include:
           - repository: pytorch/text
-            pre-script: packaging/install_torchdata.sh
+            pre-script: ""
             post-script: ""
             smoke-test-script: test/smoke_tests/smoke_tests.py
             package-name: torchtext

diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml
@@ -32,7 +32,7 @@ jobs:
       matrix:
         include:
           - repository: pytorch/text
-            pre-script: packaging/install_torchdata.sh
+            pre-script: ""
             post-script: ""
             package-name: torchtext
             smoke-test-script: test/smoke_tests/smoke_tests.py

diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml
@@ -33,7 +33,7 @@ jobs:
       matrix:
         include:
           - repository: pytorch/text
-            pre-script: packaging/install_torchdata.sh
+            pre-script: ""
             env-script: packaging/vc_env_helper.bat
             post-script: ""
             smoke-test-script: test/smoke_tests/smoke_tests.py

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -31,7 +31,6 @@ jobs:
       - name: Install Torch
         run: |
           python -m pip install cmake
-          python -m pip install --quiet --pre torch torchdata -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
           sudo ln -s /usr/bin/ninja /usr/bin/ninja-build
 
       - name: Build TorchText

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -39,15 +39,13 @@ jobs:
         python -m spacy download en_core_web_sm
         printf "* Downloading SpaCy German models\n"
         python -m spacy download de_core_news_sm
-        # Install PyTorch, Torchvision, and TorchData
+        # Install PyTorch, Torchvision
         set -ex
         conda install \
           --yes \
           -c "pytorch-${CHANNEL}" \
           -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \
           "${CUDATOOLKIT}"
-        printf "Installing torchdata nightly\n"
-        python3 -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu
         python3 setup.py develop
         # Install integration test dependencies
         python3 -m pip --quiet install parameterized

diff --git a/.github/workflows/test-linux-cpu.yml b/.github/workflows/test-linux-cpu.yml
@@ -50,16 +50,13 @@ jobs:
         printf "* Downloading SpaCy German models\n"
         python -m spacy download de_core_news_sm
 
-        # Install PyTorch, Torchvision, and TorchData
+        # Install PyTorch, Torchvision
         set -ex
         conda install \
           --yes \
           -c "pytorch-${CHANNEL}" \
           -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \
           "${CUDATOOLKIT}"
-        printf "Installing torchdata nightly\n"
-        python3 -m pip install "portalocker>=2.0.0"
-        python3 -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu
         python3 setup.py develop
         python3 -m pip install parameterized
 

diff --git a/.github/workflows/test-linux-gpu.yml b/.github/workflows/test-linux-gpu.yml
@@ -54,17 +54,14 @@ jobs:
         printf "* Downloading SpaCy German models\n"
         python -m spacy download de_core_news_sm
 
-        # Install PyTorch and TorchData
+        # Install PyTorch
         set -ex
         conda install \
           --yes \
           --quiet \
           -c "pytorch-${CHANNEL}" \
           -c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${VERSION}*"] \
           "${CUDATOOLKIT}"
-        printf "Installing torchdata nightly\n"
-        python3 -m pip install "portalocker>=2.0.0"
-        python3 -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu --quiet
         python3 setup.py develop
         python3 -m pip install parameterized --quiet
 

diff --git a/.github/workflows/test-macos-cpu.yml b/.github/workflows/test-macos-cpu.yml
@@ -55,7 +55,7 @@ jobs:
         printf "* Downloading SpaCy German models\n"
         python -m spacy download de_core_news_sm
 
-        # Install PyTorch, Torchvision, and TorchData
+        # Install PyTorch, Torchvision
         set -ex
         conda install \
           --yes \
@@ -64,9 +64,6 @@ jobs:
           "${MKL_CONSTRAINT}" \
           pytorch \
           "${CUDATOOLKIT}"
-        printf "Installing torchdata nightly\n"
-        python3 -m pip install "portalocker>=2.0.0"
-        python3 -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu
         python3 setup.py develop
         python3 -m pip install parameterized
 

diff --git a/.github/workflows/test-windows-cpu.yml b/.github/workflows/test-windows-cpu.yml
@@ -51,15 +51,12 @@ jobs:
         printf "* Downloading SpaCy German models\n"
         python -m spacy download de_core_news_sm
 
-        # Install PyTorch, Torchvision, and TorchData
+        # Install PyTorch, Torchvision
         conda install \
           --yes \
           -c "pytorch-${CHANNEL}" \
           pytorch \
           cpuonly
-        printf "Installing torchdata nightly\n"
-        python -m pip install "portalocker>=2.0.0"
-        python -m pip install --pre torchdata --index-url https://download.pytorch.org/whl/nightly/cpu
 
         printf "* Installing pywin32_postinstall script\n"
         curl --output pywin32_postinstall.py https://raw.githubusercontent.com/mhammond/pywin32/main/pywin32_postinstall.py

diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml
@@ -44,7 +44,7 @@ on:
         required: false
         type: string
       pytorch_version:
-        description: 'PyTorch version to validate (ie. 2.0, 2.2.2, etc.) - optional'
+        description: "PyTorch version to validate (ie. 2.0, 2.2.2, etc.) - optional"
         default: ""
         required: false
         type: string

diff --git a/README.rst b/README.rst
@@ -12,7 +12,7 @@
 torchtext
 +++++++++
 
-CAUTION: As of September 2023 we have paused active development of TorchText because our focus has shifted away from building out this library offering. 
+CAUTION: As of September 2023 we have paused active development of TorchText because our focus has shifted away from building out this library offering.
 We will continue to release new versions but do not anticipate any new feature development as we figure out future investments in this space.
 
 This repository consists of:

diff --git a/packaging/install_torchdata.sh b/packaging/install_torchdata.sh
diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash
@@ -190,14 +190,6 @@ setup_pip_pytorch_version() {
       -f https://download.pytorch.org/whl/torch_stable.html \
       -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html"
   fi
-  if [[ -z "$TORCHDATA_VERSION" ]]; then
-    pip_install --pre torchdata -f "https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html"
-    export TORCHDATA_VERSION="$(pip show torchdata | grep ^Version: | sed 's/Version:  *//' | sed 's/+.\+//')"
-  else
-    pip_install "torchdata==$TORCHDATA_VERSION" \
-      -f https://download.pytorch.org/whl/torch_stable.html \
-      -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html"
-  fi
 }
 
 # Fill PYTORCH_VERSION with the latest conda nightly version, and
@@ -232,10 +224,6 @@ setup_conda_pytorch_constraint() {
       export CONDA_EXTRA_BUILD_CONSTRAINT="- mkl<=2021.2.0"
     fi
   fi
-  if [[ -z "$TORCHDATA_VERSION" ]]; then
-    export TORCHDATA_VERSION="$(conda search --json 'torchdata[channel=pytorch-nightly]' | ${PYTHON} -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['torchdata'][-1]['version']))")"
-  fi
-  export CONDA_TORCHDATA_CONSTRAINT="- torchdata==$TORCHDATA_VERSION"
 }
 
 # Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT

diff --git a/packaging/torchtext/meta.yaml b/packaging/torchtext/meta.yaml
@@ -24,7 +24,6 @@ requirements:
     - requests
     - tqdm
     {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }}
-    {{ environ.get('CONDA_TORCHDATA_CONSTRAINT') }}
 
 build:
   string: py{{py}}

diff --git a/pytest.ini b/pytest.ini
@@ -1,4 +1,5 @@
 [pytest]
+addopts = --ignore-glob=test/torchtext_unittest/datasets/*
 testpaths = test/
 python_paths = ./
 markers =

diff --git a/requirements.txt b/requirements.txt
@@ -19,7 +19,6 @@ Sphinx
 pytest
 expecttest
 parameterized
-torchdata>0.5
 
 # Lets pytest find our code by automatically modifying PYTHONPATH
 pytest-pythonpath

diff --git a/setup.py b/setup.py
@@ -63,14 +63,10 @@ def _init_submodule():
 print("-- Building version " + VERSION)
 
 pytorch_package_version = os.getenv("PYTORCH_VERSION")
-torchdata_package_version = os.getenv("TORCHDATA_VERSION")
 
 pytorch_package_dep = "torch"
 if pytorch_package_version is not None:
     pytorch_package_dep += "==" + pytorch_package_version
-torchdata_package_dep = "torchdata"
-if torchdata_package_version is not None:
-    torchdata_package_dep += "==" + torchdata_package_version
 
 
 class clean(distutils.command.clean.clean):
@@ -104,7 +100,7 @@ def run(self):
     description="Text utilities, models, transforms, and datasets for PyTorch.",
     long_description=read("README.rst"),
     license="BSD",
-    install_requires=["tqdm", "requests", pytorch_package_dep, "numpy", torchdata_package_dep],
+    install_requires=["tqdm", "requests", pytorch_package_dep, "numpy"],
     python_requires=">=3.8",
     classifiers=[
         "Programming Language :: Python :: 3.8",

diff --git a/test/smoke_tests/smoke_tests.py b/test/smoke_tests/smoke_tests.py
@@ -1,28 +1,6 @@
 """Run smoke tests"""
 
-import os
-import re
-
-import torchdata
 import torchtext
-import torchtext.version  # noqa: F401
-
-NIGHTLY_ALLOWED_DELTA = 3
-channel = os.getenv("MATRIX_CHANNEL")
-
-
-def validateTorchdataVersion():
-    from datetime import datetime
-
-    date_t_str = re.findall(r"dev\d+", torchdata.__version__)[0]
-    date_t_delta = datetime.now() - datetime.strptime(date_t_str[3:], "%Y%m%d")
-
-    if date_t_delta.days >= NIGHTLY_ALLOWED_DELTA:
-        raise RuntimeError(f"torchdata binary {torchdata.__version__} is more than {NIGHTLY_ALLOWED_DELTA} days old!")
-
 
-if channel == "nightly":
-    validateTorchdataVersion()
 
 print("torchtext version is ", torchtext.__version__)
-print("torchdata version is ", torchdata.__version__)
diff --git a/torchtext/_download_hooks.py b/torchtext/_download_hooks.py
@@ -4,7 +4,6 @@
 
 # This is to allow monkey-patching in fbcode
 from torch.hub import load_state_dict_from_url  # noqa
-from torchdata.datapipes.iter import HttpReader, GDriveReader  # noqa F401
 from tqdm import tqdm