From 7853657f5c8e2add6b3a4209b9a073ab61998a32 Mon Sep 17 00:00:00 2001 From: Mike Tian-Jian Jiang Date: Tue, 17 Aug 2021 21:06:01 +0900 Subject: [PATCH 1/3] build: set to py37 with an optional Poetry env Two side notes: 1. Python 3.7.11 is for Colab; 2. Poetry is optional for managing venv and dependencies, but syncing with requirements(-dev).txt must be done manually for the time being. --- .github/workflows/code_quality.yml | 4 +- .github/workflows/test.yml | 31 + poetry.lock | 1680 ++++++++++++++++++++++++++++ pyproject.toml | 27 +- requirements.txt | 12 +- setup.py | 3 +- 6 files changed, 1747 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 poetry.lock diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index dca66d40..88143944 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -16,10 +16,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.7.11 uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.7.11 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..3a7dc9de --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,31 @@ +name: Test + +on: + push: + branches: + - master + pull_request: + branches: + - master + release: + types: + - created +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7.11 + uses: actions/setup-python@v2 + with: + python-version: 3.7.11 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install . + python -m pip install -r requirements-dev.txt + - name: Test + run: | + python -m pytest tests/test_get_dataloaders.py + python -m pytest tests/test_metadata_utils.py diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 00000000..ba101d95 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,1680 @@ +[[package]] +name = "accelerate" +version = "0.4.0" +description = "Accelerate" +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.dependencies] +pyyaml = "*" +torch = ">=1.4.0" + +[package.extras] +docs = ["docutils (==0.16.0)", "recommonmark", "sphinx (==3.2.1)", "sphinx-markdown-tables", "sphinx-rtd-theme (==0.4.3)", "sphinx-copybutton", "sphinxext-opengraph (==0.4.1)"] +quality = ["black (==21.4b0)", "isort (>=5.5.4)", "flake8 (>=3.8.3)"] +sagemaker = ["sagemaker"] +test = ["pytest", "pytest-xdist"] + +[[package]] +name = "aiohttp" +version = "3.7.4.post0" +description = "Async http client/server framework (asyncio)" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +async-timeout = ">=3.0,<4.0" +attrs = ">=17.3.0" +chardet = ">=2.0,<5.0" +multidict = ">=4.5,<7.0" +typing-extensions = ">=3.6.5" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["aiodns", "brotlipy", "cchardet"] + +[[package]] +name = "antlr4-python3-runtime" +version = "4.8" +description = "ANTLR 4.8 runtime for Python 3.7" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "async-timeout" +version = "3.0.1" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = false +python-versions = ">=3.5.3" + +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "21.2.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"] + +[[package]] +name = "black" +version = "21.8b0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +click = ">=7.1.2" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.9.0,<1" +platformdirs = ">=2" +regex = ">=2020.1.8" +tomli = ">=0.2.6,<2.0.0" +typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\""} +typing-extensions = ">=3.10.0.0" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +python2 = ["typed-ast (>=1.4.2)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "certifi" +version = "2021.5.30" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "chardet" +version = "4.0.0" +description = "Universal encoding detector for Python 2 and 3" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "charset-normalizer" +version = "2.0.4" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + +[[package]] +name = "click" +version = "8.0.1" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "configparser" +version = "5.0.2" +description = "Updated configparser from Python 3.8 for Python 2.6+." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[[package]] +name = "datasets" +version = "1.11.0" +description = "HuggingFace/Datasets is an open library of NLP datasets." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +aiohttp = {version = "*", optional = true, markers = "extra == \"streaming\""} +dill = "*" +fsspec = ">=2021.05.0" +huggingface-hub = "<0.1.0" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +multiprocess = "*" +numpy = ">=1.17" +packaging = "*" +pandas = "*" +pyarrow = ">=1.0.0,<4.0.0 || >4.0.0" +requests = ">=2.19.0" +tqdm = ">=4.42" +xxhash = "*" + +[package.extras] +apache-beam = ["apache-beam (>=2.26.0)"] +benchmarks = ["numpy (==1.18.5)", "tensorflow (==2.3.0)", "torch (==1.6.0)", "transformers (==3.0.2)"] +dev = ["absl-py", "pytest", "pytest-xdist", "aiohttp", "apache-beam (>=2.26.0)", "elasticsearch", "aiobotocore (==1.2.2)", "boto3 (==1.16.43)", "botocore (==1.19.52)", "fsspec", "moto[s3,server] (==2.0.4)", "rarfile (>=4.0)", "s3fs", "tensorflow (>=2.3)", "torch", "transformers", "bs4", "conllu", "langdetect", "lxml", "mwparserfromhell", "nltk", "openpyxl", "py7zr", "tldextract", "zstandard", "bert-score (>=0.3.6)", "rouge-score", "sacrebleu", "scipy", "seqeval", "scikit-learn", "jiwer", "sentencepiece", "toml (>=0.10.1)", "requests-file (>=1.5.1)", "tldextract (>=3.1.0)", "texttable (>=1.6.3)", "Werkzeug (>=1.0.1)", "six (>=1.15.0,<1.16.0)", "black (==21.4b0)", "flake8 (==3.7.9)", "isort", "pyyaml (>=5.3.1)", "importlib-resources"] +docs = ["docutils (==0.16.0)", "recommonmark", "sphinx (==3.1.2)", "sphinx-markdown-tables", "sphinx-rtd-theme (==0.4.3)", "sphinxext-opengraph (==0.4.1)", "sphinx-copybutton", "fsspec", "s3fs"] +quality = ["black (==21.4b0)", "flake8 (==3.7.9)", "isort", "pyyaml (>=5.3.1)"] +s3 = ["fsspec", "boto3 (==1.16.43)", "botocore (==1.19.52)", "s3fs"] +streaming = ["aiohttp"] +tensorflow = ["tensorflow (>=2.2.0)"] +tensorflow_gpu = ["tensorflow-gpu (>=2.2.0)"] +tests = ["absl-py", "pytest", "pytest-xdist", "aiohttp", "apache-beam (>=2.26.0)", "elasticsearch", "aiobotocore (==1.2.2)", "boto3 (==1.16.43)", "botocore (==1.19.52)", "fsspec", "moto[s3,server] (==2.0.4)", "rarfile (>=4.0)", "s3fs", "tensorflow (>=2.3)", "torch", "transformers", "bs4", "conllu", "langdetect", "lxml", "mwparserfromhell", "nltk", "openpyxl", "py7zr", "tldextract", "zstandard", "bert-score (>=0.3.6)", "rouge-score", "sacrebleu", "scipy", "seqeval", "scikit-learn", "jiwer", "sentencepiece", "toml (>=0.10.1)", "requests-file (>=1.5.1)", "tldextract (>=3.1.0)", "texttable (>=1.6.3)", "Werkzeug (>=1.0.1)", "six (>=1.15.0,<1.16.0)", "importlib-resources"] +torch = ["torch"] + +[[package]] +name = "dill" +version = "0.3.4" +description = "serialize all of python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*" + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + +[[package]] +name = "docker-pycreds" +version = "0.4.0" +description = "Python bindings for the docker credentials store API" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.4.0" + +[[package]] +name = "filelock" +version = "3.0.12" +description = "A platform independent file lock." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "flake8" +version = "3.9.2" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.7.0,<2.8.0" +pyflakes = ">=2.3.0,<2.4.0" + +[[package]] +name = "fsspec" +version = "2021.8.1" +description = "File-system specification" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +dask = ["dask", "distributed"] +dropbox = ["dropboxdrivefs", "requests", "dropbox"] +entrypoints = ["importlib-metadata"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +hdfs = ["pyarrow (>=1)"] +http = ["requests", "aiohttp"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] + +[[package]] +name = "gitdb" +version = "4.0.7" +description = "Git Object Database" +category = "main" +optional = false +python-versions = ">=3.4" + +[package.dependencies] +smmap = ">=3.0.1,<5" + +[[package]] +name = "gitpython" +version = "3.1.20" +description = "Python Git Library" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +gitdb = ">=4.0.1,<5" +typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.10\""} + +[[package]] +name = "huggingface-hub" +version = "0.0.16" +description = "Client library to download and publish models on the huggingface.co hub" +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.dependencies] +filelock = "*" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +packaging = ">=20.9" +requests = "*" +tqdm = "*" +typing-extensions = "*" + +[package.extras] +all = ["pytest", "black (>=20.8b1)", "isort (>=5.5.4)", "flake8 (>=3.8.3)"] +dev = ["pytest", "black (>=20.8b1)", "isort (>=5.5.4)", "flake8 (>=3.8.3)"] +quality = ["black (>=20.8b1)", "isort (>=5.5.4)", "flake8 (>=3.8.3)"] +testing = ["pytest"] +torch = ["torch"] + +[[package]] +name = "hydra-core" +version = "1.1.1" +description = "A framework for elegantly configuring complex applications" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +antlr4-python3-runtime = "4.8" +importlib-resources = {version = "*", markers = "python_version < \"3.9\""} +omegaconf = ">=2.1.0,<2.2.0" + +[[package]] +name = "idna" +version = "3.2" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "importlib-metadata" +version = "4.8.1" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +perf = ["ipython"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] + +[[package]] +name = "importlib-resources" +version = "5.2.2" +description = "Read resources from Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "isort" +version = "5.9.3" +description = "A Python utility / library to sort Python imports." +category = "dev" +optional = false +python-versions = ">=3.6.1,<4.0" + +[package.extras] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +requirements_deprecated_finder = ["pipreqs", "pip-api"] +colors = ["colorama (>=0.4.3,<0.5.0)"] +plugins = ["setuptools"] + +[[package]] +name = "joblib" +version = "1.0.1" +description = "Lightweight pipelining with Python functions" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "mccabe" +version = "0.6.1" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "multidict" +version = "5.1.0" +description = "multidict implementation" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "multiprocess" +version = "0.70.12.2" +description = "better multiprocessing and multithreading in python" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +dill = ">=0.3.4" + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "numpy" +version = "1.21.2" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.7,<3.11" + +[[package]] +name = "omegaconf" +version = "2.1.1" +description = "A flexible configuration library" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +antlr4-python3-runtime = "4.8" +PyYAML = ">=5.1.0" + +[[package]] +name = "packaging" +version = "21.0" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2" + +[[package]] +name = "pandas" +version = "1.3.2" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = false +python-versions = ">=3.7.1" + +[package.dependencies] +numpy = ">=1.17.3" +python-dateutil = ">=2.7.3" +pytz = ">=2017.3" + +[package.extras] +test = ["hypothesis (>=3.58)", "pytest (>=6.0)", "pytest-xdist"] + +[[package]] +name = "pathspec" +version = "0.9.0" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[[package]] +name = "pathtools" +version = "0.1.2" +description = "File system general utilities" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "platformdirs" +version = "2.3.0" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] +test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "promise" +version = "2.3" +description = "Promises/A+ implementation for Python" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = "*" + +[package.extras] +test = ["pytest (>=2.7.3)", "pytest-cov", "coveralls", "futures", "pytest-benchmark", "mock"] + +[[package]] +name = "protobuf" +version = "3.17.3" +description = "Protocol Buffers" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.9" + +[[package]] +name = "psutil" +version = "5.8.0" +description = "Cross-platform lib for process and system monitoring in Python." +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +test = ["ipaddress", "mock", "unittest2", "enum34", "pywin32", "wmi"] + +[[package]] +name = "py" +version = "1.10.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pyarrow" +version = "5.0.0" +description = "Python library for Apache Arrow" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pycodestyle" +version = "2.7.0" +description = "Python style guide checker" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pyflakes" +version = "2.3.1" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2021.1" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "pyyaml" +version = "5.4.1" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[[package]] +name = "regex" +version = "2021.8.28" +description = "Alternative regular expression module, to replace re." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "requests" +version = "2.26.0" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] + +[[package]] +name = "sacremoses" +version = "0.0.45" +description = "SacreMoses" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +click = "*" +joblib = "*" +regex = "*" +six = "*" +tqdm = "*" + +[[package]] +name = "sentry-sdk" +version = "1.3.1" +description = "Python client for Sentry (https://sentry.io)" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +certifi = "*" +urllib3 = ">=1.10.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.5)"] +beam = ["apache-beam (>=2.12)"] +bottle = ["bottle (>=0.12.13)"] +celery = ["celery (>=3)"] +chalice = ["chalice (>=1.16.0)"] +django = ["django (>=1.8)"] +falcon = ["falcon (>=1.4)"] +flask = ["flask (>=0.11)", "blinker (>=1.1)"] +httpx = ["httpx (>=0.16.0)"] +pure_eval = ["pure-eval", "executing", "asttokens"] +pyspark = ["pyspark (>=2.4.4)"] +rq = ["rq (>=0.6)"] +sanic = ["sanic (>=0.8)"] +sqlalchemy = ["sqlalchemy (>=1.2)"] +tornado = ["tornado (>=5)"] + +[[package]] +name = "shortuuid" +version = "1.0.1" +description = "A generator library for concise, unambiguous and URL-safe UUIDs." +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "smmap" +version = "4.0.0" +description = "A pure Python implementation of a sliding window memory map manager" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "subprocess32" +version = "3.5.4" +description = "A backport of the subprocess module from Python 3 for use on 2.x." +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4" + +[[package]] +name = "tokenizers" +version = "0.10.3" +description = "Fast and Customizable Tokenizers" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +testing = ["pytest"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tomli" +version = "1.2.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "torch" +version = "1.8.1" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +category = "main" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +numpy = "*" +typing-extensions = "*" + +[[package]] +name = "tqdm" +version = "4.62.2" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +telegram = ["requests"] + +[[package]] +name = "transformers" +version = "4.10.0" +description = "State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch" +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.dependencies] +filelock = "*" +huggingface-hub = ">=0.0.12" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +numpy = ">=1.17" +packaging = "*" +pyyaml = ">=5.1" +regex = "!=2019.12.17" +requests = "*" +sacremoses = "*" +tokenizers = ">=0.10.1,<0.11" +tqdm = ">=4.27" + +[package.extras] +all = ["tensorflow (>=2.3)", "onnxconverter-common", "keras2onnx", "torch (>=1.0)", "jax (>=0.2.8)", "jaxlib (>=0.1.65)", "flax (>=0.3.4)", "optax (>=0.0.8)", "sentencepiece (==0.1.91)", "protobuf", "tokenizers (>=0.10.1,<0.11)", "torchaudio", "soundfile", "pillow", "optuna", "ray", "timm", "codecarbon (==1.2.0)"] +audio = ["soundfile"] +codecarbon = ["codecarbon (==1.2.0)"] +deepspeed = ["deepspeed (>=0.5.1)"] +dev = ["tensorflow (>=2.3)", "onnxconverter-common", "keras2onnx", "torch (>=1.0)", "jax (>=0.2.8)", "jaxlib (>=0.1.65)", "flax (>=0.3.4)", "optax (>=0.0.8)", "sentencepiece (==0.1.91)", "protobuf", "tokenizers (>=0.10.1,<0.11)", "torchaudio", "soundfile", "pillow", "optuna", "ray", "timm", "codecarbon (==1.2.0)", "pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil", "datasets", "pytest-timeout", "black (==21.4b0)", "sacrebleu (>=1.4.12,<2.0.0)", "rouge-score", "nltk", "GitPython (<3.1.19)", "faiss-cpu", "cookiecutter (==1.7.2)", "isort (>=5.5.4)", "flake8 (>=3.8.3)", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "unidic-lite (>=1.0.7)", "unidic (>=1.0.2)", "docutils (==0.16.0)", "recommonmark", "sphinx (==3.5.4)", "sphinx-markdown-tables", "sphinx-rtd-theme (==0.4.3)", "sphinx-copybutton", "sphinxext-opengraph (==0.4.1)", "sphinx-intl", "scikit-learn"] +docs = ["tensorflow (>=2.3)", "onnxconverter-common", "keras2onnx", "torch (>=1.0)", "jax (>=0.2.8)", "jaxlib (>=0.1.65)", "flax (>=0.3.4)", "optax (>=0.0.8)", "sentencepiece (==0.1.91)", "protobuf", "tokenizers (>=0.10.1,<0.11)", "torchaudio", "soundfile", "pillow", "optuna", "ray", "timm", "codecarbon (==1.2.0)", "docutils (==0.16.0)", "recommonmark", "sphinx (==3.5.4)", "sphinx-markdown-tables", "sphinx-rtd-theme (==0.4.3)", "sphinx-copybutton", "sphinxext-opengraph (==0.4.1)", "sphinx-intl"] +docs_specific = ["docutils (==0.16.0)", "recommonmark", "sphinx (==3.5.4)", "sphinx-markdown-tables", "sphinx-rtd-theme (==0.4.3)", "sphinx-copybutton", "sphinxext-opengraph (==0.4.1)", "sphinx-intl"] +fairscale = ["fairscale (>0.3)"] +flax = ["jax (>=0.2.8)", "jaxlib (>=0.1.65)", "flax (>=0.3.4)", "optax (>=0.0.8)"] +flax-speech = ["soundfile"] +integrations = ["optuna", "ray"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "unidic-lite (>=1.0.7)", "unidic (>=1.0.2)"] +modelcreation = ["cookiecutter (==1.7.2)"] +onnx = ["onnxconverter-common", "keras2onnx", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] +onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] +optuna = ["optuna"] +quality = ["black (==21.4b0)", "isort (>=5.5.4)", "flake8 (>=3.8.3)"] +ray = ["ray"] +retrieval = ["faiss-cpu", "datasets"] +sagemaker = ["sagemaker (>=2.31.0)"] +sentencepiece = ["sentencepiece (==0.1.91)", "protobuf"] +serving = ["pydantic", "uvicorn", "fastapi", "starlette"] +sklearn = ["scikit-learn"] +speech = ["torchaudio", "soundfile"] +testing = ["pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil", "datasets", "pytest-timeout", "black (==21.4b0)", "sacrebleu (>=1.4.12,<2.0.0)", "rouge-score", "nltk", "GitPython (<3.1.19)", "faiss-cpu", "cookiecutter (==1.7.2)"] +tf = ["tensorflow (>=2.3)", "onnxconverter-common", "keras2onnx"] +tf-cpu = ["tensorflow-cpu (>=2.3)", "onnxconverter-common", "keras2onnx"] +tf-speech = ["soundfile"] +timm = ["timm"] +tokenizers = ["tokenizers (>=0.10.1,<0.11)"] +torch = ["torch (>=1.0)"] +torch-speech = ["torchaudio", "soundfile"] +torchhub = ["filelock", "huggingface-hub (>=0.0.12)", "importlib-metadata", "numpy (>=1.17)", "packaging", "protobuf", "regex (!=2019.12.17)", "requests", "sacremoses", "sentencepiece (==0.1.91)", "torch (>=1.0)", "tokenizers (>=0.10.1,<0.11)", "tqdm (>=4.27)"] +vision = ["pillow"] + +[[package]] +name = "typed-ast" +version = "1.4.3" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "typing-extensions" +version = "3.10.0.2" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "urllib3" +version = "1.26.6" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "wandb" +version = "0.12.1" +description = "A CLI and library for interacting with the Weights and Biases API." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +Click = ">=7.0,<8.0.0 || >8.0.0" +configparser = ">=3.8.1" +docker-pycreds = ">=0.4.0" +GitPython = ">=1.0.0" +pathtools = "*" +promise = ">=2.0,<3" +protobuf = ">=3.12.0" +psutil = ">=5.0.0" +python-dateutil = ">=2.6.1" +PyYAML = "*" +requests = ">=2.0.0,<3" +sentry-sdk = ">=1.0.0" +shortuuid = ">=0.5.0" +six = ">=1.13.0" +subprocess32 = ">=3.5.3" + +[package.extras] +aws = ["boto3"] +gcp = ["google-cloud-storage"] +grpc = ["grpcio (==1.27.2)"] +kubeflow = ["kubernetes", "minio", "google-cloud-storage", "sh"] +launch = ["docker", "jupyter-repo2docker", "chardet", "iso8601"] +media = ["numpy", "moviepy", "pillow", "bokeh", "soundfile", "plotly"] +sweeps = ["numpy (>=1.15,<1.21)", "scipy (>=1.5.4)", "pyyaml", "scikit-learn (==0.24.1)", "jsonschema (>=3.2.0)", "jsonref (>=0.2)", "pydantic (>=1.8.2)"] + +[[package]] +name = "xxhash" +version = "2.0.2" +description = "Python binding for xxHash" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "yarl" +version = "1.6.3" +description = "Yet another URL library" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" +typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} + +[[package]] +name = "zipp" +version = "3.5.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[metadata] +lock-version = "1.1" +python-versions = ">=3.7.11, <3.10" +content-hash = "da5214fad0b66ebf5a4f52c4b8e5b87139e161371e10e290ce7f122e451bde59" + +[metadata.files] +accelerate = [ + {file = "accelerate-0.4.0-py3-none-any.whl", hash = "sha256:08e37ce6d314e79aaf2ef01782564b7588f7163623fc01bfd2b57e226d579453"}, + {file = "accelerate-0.4.0.tar.gz", hash = "sha256:4f1c5153ab855943662b02a648e038f4fc9255525f03b2494534527e7339dd07"}, +] +aiohttp = [ + {file = "aiohttp-3.7.4.post0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:3cf75f7cdc2397ed4442594b935a11ed5569961333d49b7539ea741be2cc79d5"}, + {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:4b302b45040890cea949ad092479e01ba25911a15e648429c7c5aae9650c67a8"}, + {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:fe60131d21b31fd1a14bd43e6bb88256f69dfc3188b3a89d736d6c71ed43ec95"}, + {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:393f389841e8f2dfc86f774ad22f00923fdee66d238af89b70ea314c4aefd290"}, + {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:c6e9dcb4cb338d91a73f178d866d051efe7c62a7166653a91e7d9fb18274058f"}, + {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:5df68496d19f849921f05f14f31bd6ef53ad4b00245da3195048c69934521809"}, + {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:0563c1b3826945eecd62186f3f5c7d31abb7391fedc893b7e2b26303b5a9f3fe"}, + {file = "aiohttp-3.7.4.post0-cp36-cp36m-win32.whl", hash = "sha256:3d78619672183be860b96ed96f533046ec97ca067fd46ac1f6a09cd9b7484287"}, + {file = "aiohttp-3.7.4.post0-cp36-cp36m-win_amd64.whl", hash = "sha256:f705e12750171c0ab4ef2a3c76b9a4024a62c4103e3a55dd6f99265b9bc6fcfc"}, + {file = "aiohttp-3.7.4.post0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:230a8f7e24298dea47659251abc0fd8b3c4e38a664c59d4b89cca7f6c09c9e87"}, + {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:2e19413bf84934d651344783c9f5e22dee452e251cfd220ebadbed2d9931dbf0"}, + {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:e4b2b334e68b18ac9817d828ba44d8fcb391f6acb398bcc5062b14b2cbeac970"}, + {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:d012ad7911653a906425d8473a1465caa9f8dea7fcf07b6d870397b774ea7c0f"}, + {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:40eced07f07a9e60e825554a31f923e8d3997cfc7fb31dbc1328c70826e04cde"}, + {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:209b4a8ee987eccc91e2bd3ac36adee0e53a5970b8ac52c273f7f8fd4872c94c"}, + {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:14762875b22d0055f05d12abc7f7d61d5fd4fe4642ce1a249abdf8c700bf1fd8"}, + {file = "aiohttp-3.7.4.post0-cp37-cp37m-win32.whl", hash = "sha256:7615dab56bb07bff74bc865307aeb89a8bfd9941d2ef9d817b9436da3a0ea54f"}, + {file = "aiohttp-3.7.4.post0-cp37-cp37m-win_amd64.whl", hash = "sha256:d9e13b33afd39ddeb377eff2c1c4f00544e191e1d1dee5b6c51ddee8ea6f0cf5"}, + {file = "aiohttp-3.7.4.post0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:547da6cacac20666422d4882cfcd51298d45f7ccb60a04ec27424d2f36ba3eaf"}, + {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:af9aa9ef5ba1fd5b8c948bb11f44891968ab30356d65fd0cc6707d989cd521df"}, + {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:64322071e046020e8797117b3658b9c2f80e3267daec409b350b6a7a05041213"}, + {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:bb437315738aa441251214dad17428cafda9cdc9729499f1d6001748e1d432f4"}, + {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:e54962802d4b8b18b6207d4a927032826af39395a3bd9196a5af43fc4e60b009"}, + {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:a00bb73540af068ca7390e636c01cbc4f644961896fa9363154ff43fd37af2f5"}, + {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:79ebfc238612123a713a457d92afb4096e2148be17df6c50fb9bf7a81c2f8013"}, + {file = "aiohttp-3.7.4.post0-cp38-cp38-win32.whl", hash = "sha256:515dfef7f869a0feb2afee66b957cc7bbe9ad0cdee45aec7fdc623f4ecd4fb16"}, + {file = "aiohttp-3.7.4.post0-cp38-cp38-win_amd64.whl", hash = "sha256:114b281e4d68302a324dd33abb04778e8557d88947875cbf4e842c2c01a030c5"}, + {file = "aiohttp-3.7.4.post0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:7b18b97cf8ee5452fa5f4e3af95d01d84d86d32c5e2bfa260cf041749d66360b"}, + {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:15492a6368d985b76a2a5fdd2166cddfea5d24e69eefed4630cbaae5c81d89bd"}, + {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bdb230b4943891321e06fc7def63c7aace16095be7d9cf3b1e01be2f10fba439"}, + {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:cffe3ab27871bc3ea47df5d8f7013945712c46a3cc5a95b6bee15887f1675c22"}, + {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:f881853d2643a29e643609da57b96d5f9c9b93f62429dcc1cbb413c7d07f0e1a"}, + {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:a5ca29ee66f8343ed336816c553e82d6cade48a3ad702b9ffa6125d187e2dedb"}, + {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:17c073de315745a1510393a96e680d20af8e67e324f70b42accbd4cb3315c9fb"}, + {file = "aiohttp-3.7.4.post0-cp39-cp39-win32.whl", hash = "sha256:932bb1ea39a54e9ea27fc9232163059a0b8855256f4052e776357ad9add6f1c9"}, + {file = "aiohttp-3.7.4.post0-cp39-cp39-win_amd64.whl", hash = "sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe"}, + {file = "aiohttp-3.7.4.post0.tar.gz", hash = "sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf"}, +] +antlr4-python3-runtime = [ + {file = "antlr4-python3-runtime-4.8.tar.gz", hash = "sha256:15793f5d0512a372b4e7d2284058ad32ce7dd27126b105fb0b2245130445db33"}, +] +async-timeout = [ + {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, + {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, +] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"}, + {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"}, +] +black = [ + {file = "black-21.8b0-py3-none-any.whl", hash = "sha256:2a0f9a8c2b2a60dbcf1ccb058842fb22bdbbcb2f32c6cc02d9578f90b92ce8b7"}, + {file = "black-21.8b0.tar.gz", hash = "sha256:570608d28aa3af1792b98c4a337dbac6367877b47b12b88ab42095cfc1a627c2"}, +] +certifi = [ + {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, + {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, +] +chardet = [ + {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, + {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"}, + {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"}, +] +click = [ + {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, + {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +configparser = [ + {file = "configparser-5.0.2-py3-none-any.whl", hash = "sha256:af59f2cdd7efbdd5d111c1976ecd0b82db9066653362f0962d7bf1d3ab89a1fa"}, + {file = "configparser-5.0.2.tar.gz", hash = "sha256:85d5de102cfe6d14a5172676f09d19c465ce63d6019cf0a4ef13385fc535e828"}, +] +datasets = [ + {file = "datasets-1.11.0-py3-none-any.whl", hash = "sha256:603612b018794e33d8f0655235731bc139b141cb8f864c2f29140940da16955f"}, + {file = "datasets-1.11.0.tar.gz", hash = "sha256:3b01bf12951903e83b528d41129876426eb3a5fbcaf2645552283330528c92bf"}, +] +dill = [ + {file = "dill-0.3.4-py2.py3-none-any.whl", hash = "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f"}, + {file = "dill-0.3.4.zip", hash = "sha256:9f9734205146b2b353ab3fec9af0070237b6ddae78452af83d2fca84d739e675"}, +] +docker-pycreds = [ + {file = "docker-pycreds-0.4.0.tar.gz", hash = "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4"}, + {file = "docker_pycreds-0.4.0-py2.py3-none-any.whl", hash = "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49"}, +] +filelock = [ + {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"}, + {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, +] +flake8 = [ + {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, + {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, +] +fsspec = [ + {file = "fsspec-2021.8.1-py3-none-any.whl", hash = "sha256:30f27c059a414d1f434b14b2d0e75c8d9c3dd473ad8daeccb444d9d4069b9f03"}, + {file = "fsspec-2021.8.1.tar.gz", hash = "sha256:af125917788b77782899bbd4484d29e5407f53d2bb04cdfa025fe4931201a555"}, +] +gitdb = [ + {file = "gitdb-4.0.7-py3-none-any.whl", hash = "sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0"}, + {file = "gitdb-4.0.7.tar.gz", hash = "sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005"}, +] +gitpython = [ + {file = "GitPython-3.1.20-py3-none-any.whl", hash = "sha256:b1e1c269deab1b08ce65403cf14e10d2ef1f6c89e33ea7c5e5bb0222ea593b8a"}, + {file = "GitPython-3.1.20.tar.gz", hash = "sha256:df0e072a200703a65387b0cfdf0466e3bab729c0458cf6b7349d0e9877636519"}, +] +huggingface-hub = [ + {file = "huggingface_hub-0.0.16-py3-none-any.whl", hash = "sha256:c8170998f1ac43ec9253f5fd321213aeee54a9b938c9ce5e696a06274710b67c"}, + {file = "huggingface_hub-0.0.16.tar.gz", hash = "sha256:6742126aef62244db6dc4cd0d4d9f13eef6a9b125c972f92111fb285c1100ae6"}, +] +hydra-core = [ + {file = "hydra-core-1.1.1.tar.gz", hash = "sha256:d396d4d979ff66e2d654b8ef5435ad3b6bbd093c4330f3ed316c339d5d741269"}, + {file = "hydra_core-1.1.1-py3-none-any.whl", hash = "sha256:928b154d8105db4ee67d768e3cd0f74ca9373121fc99dee54488393b17a31929"}, +] +idna = [ + {file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"}, + {file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"}, +] +importlib-metadata = [ + {file = "importlib_metadata-4.8.1-py3-none-any.whl", hash = "sha256:b618b6d2d5ffa2f16add5697cf57a46c76a56229b0ed1c438322e4e95645bd15"}, + {file = "importlib_metadata-4.8.1.tar.gz", hash = "sha256:f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1"}, +] +importlib-resources = [ + {file = "importlib_resources-5.2.2-py3-none-any.whl", hash = "sha256:2480d8e07d1890056cb53c96e3de44fead9c62f2ba949b0f2e4c4345f4afa977"}, + {file = "importlib_resources-5.2.2.tar.gz", hash = "sha256:a65882a4d0fe5fbf702273456ba2ce74fe44892c25e42e057aca526b702a6d4b"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +isort = [ + {file = "isort-5.9.3-py3-none-any.whl", hash = "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2"}, + {file = "isort-5.9.3.tar.gz", hash = "sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899"}, +] +joblib = [ + {file = "joblib-1.0.1-py3-none-any.whl", hash = "sha256:feeb1ec69c4d45129954f1b7034954241eedfd6ba39b5e9e4b6883be3332d5e5"}, + {file = "joblib-1.0.1.tar.gz", hash = "sha256:9c17567692206d2f3fb9ecf5e991084254fe631665c450b443761c4186a613f7"}, +] +mccabe = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] +multidict = [ + {file = "multidict-5.1.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f"}, + {file = "multidict-5.1.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf"}, + {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281"}, + {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d"}, + {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d"}, + {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da"}, + {file = "multidict-5.1.0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224"}, + {file = "multidict-5.1.0-cp36-cp36m-win32.whl", hash = "sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26"}, + {file = "multidict-5.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6"}, + {file = "multidict-5.1.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76"}, + {file = "multidict-5.1.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a"}, + {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f"}, + {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348"}, + {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93"}, + {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9"}, + {file = "multidict-5.1.0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37"}, + {file = "multidict-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5"}, + {file = "multidict-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632"}, + {file = "multidict-5.1.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952"}, + {file = "multidict-5.1.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79"}, + {file = "multidict-5.1.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456"}, + {file = "multidict-5.1.0-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7"}, + {file = "multidict-5.1.0-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635"}, + {file = "multidict-5.1.0-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a"}, + {file = "multidict-5.1.0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea"}, + {file = "multidict-5.1.0-cp38-cp38-win32.whl", hash = "sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656"}, + {file = "multidict-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3"}, + {file = "multidict-5.1.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93"}, + {file = "multidict-5.1.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647"}, + {file = "multidict-5.1.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d"}, + {file = "multidict-5.1.0-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8"}, + {file = "multidict-5.1.0-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1"}, + {file = "multidict-5.1.0-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841"}, + {file = "multidict-5.1.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda"}, + {file = "multidict-5.1.0-cp39-cp39-win32.whl", hash = "sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80"}, + {file = "multidict-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359"}, + {file = "multidict-5.1.0.tar.gz", hash = "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5"}, +] +multiprocess = [ + {file = "multiprocess-0.70.12.2-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:35d41e410ca2a32977a483ae1f40f86b193b45cecf85567c2fae402fb8bf172e"}, + {file = "multiprocess-0.70.12.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:9a02237eae21975155c816883479f72e239d16823a6bc063173d59acec9bcf41"}, + {file = "multiprocess-0.70.12.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:f12a939cd2f01d0a900e7ef2aaee3c351a49fd2297d7f760b537af22727561b8"}, + {file = "multiprocess-0.70.12.2-cp27-cp27m-win32.whl", hash = "sha256:be3ad3eaf204abc646d85e70e41244f66d88200628a0ab867c8fc206b97cedbf"}, + {file = "multiprocess-0.70.12.2-cp27-cp27m-win_amd64.whl", hash = "sha256:c85ffc38c50c5a4f32f3f3c1a284725b7b5040188f254eba6e572c53d3da525b"}, + {file = "multiprocess-0.70.12.2-pp27-none-any.whl", hash = "sha256:a9f58945edb234591684c0a181b744a3231643814ef3a8f47cea9a2073b4b2bb"}, + {file = "multiprocess-0.70.12.2-pp36-none-any.whl", hash = "sha256:0e0a5ae4bd84e4c22baddf824d3b8168214f8c1cce51e2cb080421cb1f7b04d1"}, + {file = "multiprocess-0.70.12.2-pp37-none-any.whl", hash = "sha256:916a314a1e0f3454033d59672ba6181fa45948ab1091d68cdd479258576e7b27"}, + {file = "multiprocess-0.70.12.2-py36-none-any.whl", hash = "sha256:b3f866f7d9c7acc1a9cb1b6063a29f5cb140ff545b35b71fd4bfdac6f19d75fa"}, + {file = "multiprocess-0.70.12.2-py37-none-any.whl", hash = "sha256:6aa67e805e50b6e9dfc56dd0f0c85ac3409e6791d4ec5405c5f9bc0a47d745a4"}, + {file = "multiprocess-0.70.12.2-py38-none-any.whl", hash = "sha256:85941e650c277af44fc82e3e97faacb920e5ce3615238b540cbad4012d6f60e9"}, + {file = "multiprocess-0.70.12.2-py39-none-any.whl", hash = "sha256:6f812a1d3f198b7cacd63983f60e2dc1338bd4450893f90c435067b5a3127e6f"}, + {file = "multiprocess-0.70.12.2.zip", hash = "sha256:206bb9b97b73f87fec1ed15a19f8762950256aa84225450abc7150d02855a083"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +numpy = [ + {file = "numpy-1.21.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13"}, + {file = "numpy-1.21.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218"}, + {file = "numpy-1.21.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf"}, + {file = "numpy-1.21.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1"}, + {file = "numpy-1.21.2-cp37-cp37m-win32.whl", hash = "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310"}, + {file = "numpy-1.21.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496"}, + {file = "numpy-1.21.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3"}, + {file = "numpy-1.21.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2"}, + {file = "numpy-1.21.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931"}, + {file = "numpy-1.21.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f"}, + {file = "numpy-1.21.2-cp38-cp38-win32.whl", hash = "sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557"}, + {file = "numpy-1.21.2-cp38-cp38-win_amd64.whl", hash = "sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0"}, + {file = "numpy-1.21.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38"}, + {file = "numpy-1.21.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d"}, + {file = "numpy-1.21.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419"}, + {file = "numpy-1.21.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723"}, + {file = "numpy-1.21.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737"}, + {file = "numpy-1.21.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f"}, + {file = "numpy-1.21.2-cp39-cp39-win32.whl", hash = "sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad"}, + {file = "numpy-1.21.2-cp39-cp39-win_amd64.whl", hash = "sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724"}, + {file = "numpy-1.21.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7"}, + {file = "numpy-1.21.2.zip", hash = "sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc"}, +] +omegaconf = [ + {file = "omegaconf-2.1.1-py3-none-any.whl", hash = "sha256:be93d73eaa2564fbe52d88ee13e3b79f4c6e04876b2f326551a21391f7dc6367"}, + {file = "omegaconf-2.1.1.tar.gz", hash = "sha256:c65e05530369484e074a24038fe31812c73561aa9d916abfd1209e4073136ae5"}, +] +packaging = [ + {file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"}, + {file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"}, +] +pandas = [ + {file = "pandas-1.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ba7ceb8abc6dbdb1e34612d1173d61e4941f1a1eb7e6f703b2633134ae6a6c89"}, + {file = "pandas-1.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcb71b1935249de80e3a808227189eee381d4d74a31760ced2df21eedc92a8e3"}, + {file = "pandas-1.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa54dc1d3e5d004a09ab0b1751473698011ddf03e14f1f59b84ad9a6ac630975"}, + {file = "pandas-1.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34ced9ce5d5b17b556486da7256961b55b471d64a8990b56e67a84ebeb259416"}, + {file = "pandas-1.3.2-cp37-cp37m-win32.whl", hash = "sha256:a56246de744baf646d1f3e050c4653d632bc9cd2e0605f41051fea59980e880a"}, + {file = "pandas-1.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:53b17e4debba26b7446b1e4795c19f94f0c715e288e08145e44bdd2865e819b3"}, + {file = "pandas-1.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f07a9745ca075ae73a5ce116f5e58f691c0dc9de0bff163527858459df5c176f"}, + {file = "pandas-1.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9e8e0ce5284ebebe110efd652c164ed6eab77f5de4c3533abc756302ee77765"}, + {file = "pandas-1.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59a78d7066d1c921a77e3306aa0ebf6e55396c097d5dfcc4df8defe3dcecb735"}, + {file = "pandas-1.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:132def05e73d292c949b02e7ef873debb77acc44a8b119d215921046f0c3a91d"}, + {file = "pandas-1.3.2-cp38-cp38-win32.whl", hash = "sha256:69e1b2f5811f46827722fd641fdaeedb26002bd1e504eacc7a8ec36bdc25393e"}, + {file = "pandas-1.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:7996d311413379136baf0f3cf2a10e331697657c87ced3f17ac7c77f77fe34a3"}, + {file = "pandas-1.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1738154049062156429a5cf2fd79a69c9f3fa4f231346a7ec6fd156cd1a9a621"}, + {file = "pandas-1.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cce01f6d655b4add966fcd36c32c5d1fe84628e200626b3f5e2f40db2d16a0f"}, + {file = "pandas-1.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1099e2a0cd3a01ec62cca183fc1555833a2d43764950ef8cb5948c8abfc51014"}, + {file = "pandas-1.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cd5776be891331a3e6b425b5abeab9596abea18435c5982191356f9b24ae731"}, + {file = "pandas-1.3.2-cp39-cp39-win32.whl", hash = "sha256:66a95361b81b4ba04b699ecd2416b0591f40cd1e24c60a8bfe0d19009cfa575a"}, + {file = "pandas-1.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:89f40e5d21814192802421df809f948247d39ffe171e45fe2ab4abf7bd4279d8"}, + {file = "pandas-1.3.2.tar.gz", hash = "sha256:cbcb84d63867af3411fa063af3de64902665bb5b3d40b25b2059e40603594e87"}, +] +pathspec = [ + {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, + {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, +] +pathtools = [ + {file = "pathtools-0.1.2.tar.gz", hash = "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0"}, +] +platformdirs = [ + {file = "platformdirs-2.3.0-py3-none-any.whl", hash = "sha256:8003ac87717ae2c7ee1ea5a84a1a61e87f3fbd16eb5aadba194ea30a9019f648"}, + {file = "platformdirs-2.3.0.tar.gz", hash = "sha256:15b056538719b1c94bdaccb29e5f81879c7f7f0f4a153f46086d155dffcd4f0f"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +promise = [ + {file = "promise-2.3.tar.gz", hash = "sha256:dfd18337c523ba4b6a58801c164c1904a9d4d1b1747c7d5dbf45b693a49d93d0"}, +] +protobuf = [ + {file = "protobuf-3.17.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ab6bb0e270c6c58e7ff4345b3a803cc59dbee19ddf77a4719c5b635f1d547aa8"}, + {file = "protobuf-3.17.3-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:13ee7be3c2d9a5d2b42a1030976f760f28755fcf5863c55b1460fd205e6cd637"}, + {file = "protobuf-3.17.3-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:1556a1049ccec58c7855a78d27e5c6e70e95103b32de9142bae0576e9200a1b0"}, + {file = "protobuf-3.17.3-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:f0e59430ee953184a703a324b8ec52f571c6c4259d496a19d1cabcdc19dabc62"}, + {file = "protobuf-3.17.3-cp35-cp35m-win32.whl", hash = "sha256:a981222367fb4210a10a929ad5983ae93bd5a050a0824fc35d6371c07b78caf6"}, + {file = "protobuf-3.17.3-cp35-cp35m-win_amd64.whl", hash = "sha256:6d847c59963c03fd7a0cd7c488cadfa10cda4fff34d8bc8cba92935a91b7a037"}, + {file = "protobuf-3.17.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:145ce0af55c4259ca74993ddab3479c78af064002ec8227beb3d944405123c71"}, + {file = "protobuf-3.17.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ce4d8bf0321e7b2d4395e253f8002a1a5ffbcfd7bcc0a6ba46712c07d47d0b4"}, + {file = "protobuf-3.17.3-cp36-cp36m-win32.whl", hash = "sha256:7a4c97961e9e5b03a56f9a6c82742ed55375c4a25f2692b625d4087d02ed31b9"}, + {file = "protobuf-3.17.3-cp36-cp36m-win_amd64.whl", hash = "sha256:a22b3a0dbac6544dacbafd4c5f6a29e389a50e3b193e2c70dae6bbf7930f651d"}, + {file = "protobuf-3.17.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ffea251f5cd3c0b9b43c7a7a912777e0bc86263436a87c2555242a348817221b"}, + {file = "protobuf-3.17.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:9b7a5c1022e0fa0dbde7fd03682d07d14624ad870ae52054849d8960f04bc764"}, + {file = "protobuf-3.17.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8727ee027157516e2c311f218ebf2260a18088ffb2d29473e82add217d196b1c"}, + {file = "protobuf-3.17.3-cp37-cp37m-win32.whl", hash = "sha256:14c1c9377a7ffbeaccd4722ab0aa900091f52b516ad89c4b0c3bb0a4af903ba5"}, + {file = "protobuf-3.17.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c56c050a947186ba51de4f94ab441d7f04fcd44c56df6e922369cc2e1a92d683"}, + {file = "protobuf-3.17.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2ae692bb6d1992afb6b74348e7bb648a75bb0d3565a3f5eea5bec8f62bd06d87"}, + {file = "protobuf-3.17.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:99938f2a2d7ca6563c0ade0c5ca8982264c484fdecf418bd68e880a7ab5730b1"}, + {file = "protobuf-3.17.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6902a1e4b7a319ec611a7345ff81b6b004b36b0d2196ce7a748b3493da3d226d"}, + {file = "protobuf-3.17.3-cp38-cp38-win32.whl", hash = "sha256:59e5cf6b737c3a376932fbfb869043415f7c16a0cf176ab30a5bbc419cd709c1"}, + {file = "protobuf-3.17.3-cp38-cp38-win_amd64.whl", hash = "sha256:ebcb546f10069b56dc2e3da35e003a02076aaa377caf8530fe9789570984a8d2"}, + {file = "protobuf-3.17.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ffbd23640bb7403574f7aff8368e2aeb2ec9a5c6306580be48ac59a6bac8bde"}, + {file = "protobuf-3.17.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:26010f693b675ff5a1d0e1bdb17689b8b716a18709113288fead438703d45539"}, + {file = "protobuf-3.17.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e76d9686e088fece2450dbc7ee905f9be904e427341d289acbe9ad00b78ebd47"}, + {file = "protobuf-3.17.3-cp39-cp39-win32.whl", hash = "sha256:a38bac25f51c93e4be4092c88b2568b9f407c27217d3dd23c7a57fa522a17554"}, + {file = "protobuf-3.17.3-cp39-cp39-win_amd64.whl", hash = "sha256:85d6303e4adade2827e43c2b54114d9a6ea547b671cb63fafd5011dc47d0e13d"}, + {file = "protobuf-3.17.3-py2.py3-none-any.whl", hash = "sha256:2bfb815216a9cd9faec52b16fd2bfa68437a44b67c56bee59bc3926522ecb04e"}, + {file = "protobuf-3.17.3.tar.gz", hash = "sha256:72804ea5eaa9c22a090d2803813e280fb273b62d5ae497aaf3553d141c4fdd7b"}, +] +psutil = [ + {file = "psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"}, + {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c"}, + {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df"}, + {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131"}, + {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60"}, + {file = "psutil-5.8.0-cp27-none-win32.whl", hash = "sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876"}, + {file = "psutil-5.8.0-cp27-none-win_amd64.whl", hash = "sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65"}, + {file = "psutil-5.8.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8"}, + {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6"}, + {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac"}, + {file = "psutil-5.8.0-cp36-cp36m-win32.whl", hash = "sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2"}, + {file = "psutil-5.8.0-cp36-cp36m-win_amd64.whl", hash = "sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d"}, + {file = "psutil-5.8.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935"}, + {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d"}, + {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023"}, + {file = "psutil-5.8.0-cp37-cp37m-win32.whl", hash = "sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394"}, + {file = "psutil-5.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563"}, + {file = "psutil-5.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef"}, + {file = "psutil-5.8.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28"}, + {file = "psutil-5.8.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b"}, + {file = "psutil-5.8.0-cp38-cp38-win32.whl", hash = "sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d"}, + {file = "psutil-5.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d"}, + {file = "psutil-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7"}, + {file = "psutil-5.8.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4"}, + {file = "psutil-5.8.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b"}, + {file = "psutil-5.8.0-cp39-cp39-win32.whl", hash = "sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0"}, + {file = "psutil-5.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3"}, + {file = "psutil-5.8.0.tar.gz", hash = "sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6"}, +] +py = [ + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, +] +pyarrow = [ + {file = "pyarrow-5.0.0-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:e9ec80f4a77057498cf4c5965389e42e7f6a618b6859e6dd615e57505c9167a6"}, + {file = "pyarrow-5.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b1453c2411b5062ba6bf6832dbc4df211ad625f678c623a2ee177aee158f199b"}, + {file = "pyarrow-5.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:9e04d3621b9f2f23898eed0d044203f66c156d880f02c5534a7f9947ebb1a4af"}, + {file = "pyarrow-5.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:64f30aa6b28b666a925d11c239344741850eb97c29d3aa0f7187918cf82494f7"}, + {file = "pyarrow-5.0.0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:99c8b0f7e2ce2541dd4c0c0101d9944bb8e592ae3295fe7a2f290ab99222666d"}, + {file = "pyarrow-5.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:456a4488ae810a0569d1adf87dbc522bcc9a0e4a8d1809b934ca28c163d8edce"}, + {file = "pyarrow-5.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:c5493d2414d0d690a738aac8dd6d38518d1f9b870e52e24f89d8d7eb3afd4161"}, + {file = "pyarrow-5.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1832709281efefa4f199c639e9f429678286329860188e53beeda71750775923"}, + {file = "pyarrow-5.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:b6387d2058d95fa48ccfedea810a768187affb62f4a3ef6595fa30bf9d1a65cf"}, + {file = "pyarrow-5.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:bbe2e439bec2618c74a3bb259700c8a7353dc2ea0c5a62686b6cf04a50ab1e0d"}, + {file = "pyarrow-5.0.0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:5c0d1b68e67bb334a5af0cecdf9b6a702aaa4cc259c5cbb71b25bbed40fcedaf"}, + {file = "pyarrow-5.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:6e937ce4a40ea0cc7896faff96adecadd4485beb53fbf510b46858e29b2e75ae"}, + {file = "pyarrow-5.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:7560332e5846f0e7830b377c14c93624e24a17f91c98f0b25dafb0ca1ea6ba02"}, + {file = "pyarrow-5.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:53e550dec60d1ab86cba3afa1719dc179a8bc9632a0e50d9fe91499cf0a7f2bc"}, + {file = "pyarrow-5.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2d26186ca9748a1fb89ae6c1fa04fb343a4279b53f118734ea8096f15d66c820"}, + {file = "pyarrow-5.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:7c4edd2bacee3eea6c8c28bddb02347f9d41a55ec9692c71c6de6e47c62a7f0d"}, + {file = "pyarrow-5.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:601b0aabd6fb066429e706282934d4d8d38f53bdb8d82da9576be49f07eedf5c"}, + {file = "pyarrow-5.0.0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:ff21711f6ff3b0bc90abc8ca8169e676faeb2401ddc1a0bc1c7dc181708a3406"}, + {file = "pyarrow-5.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:ed135a99975380c27077f9d0e210aea8618ed9fadcec0e71f8a3190939557afe"}, + {file = "pyarrow-5.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:6e1f0e4374061116f40e541408a8a170c170d0a070b788717e18165ebfdd2a54"}, + {file = "pyarrow-5.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:4341ac0f552dc04c450751e049976940c7f4f8f2dae03685cc465ebe0a61e231"}, + {file = "pyarrow-5.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c3fc856f107ca2fb3c9391d7ea33bbb33f3a1c2b4a0e2b41f7525c626214cc03"}, + {file = "pyarrow-5.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:357605665fbefb573d40939b13a684c2490b6ed1ab4a5de8dd246db4ab02e5a4"}, + {file = "pyarrow-5.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f4db312e9ba80e730cefcae0a05b63ea5befc7634c28df56682b628ad8e1c25c"}, + {file = "pyarrow-5.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1d9485741e497ccc516cb0a0c8f56e22be55aea815be185c3f9a681323b0e614"}, + {file = "pyarrow-5.0.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:b3115df938b8d7a7372911a3cb3904196194bcea8bb48911b4b3eafee3ab8d90"}, + {file = "pyarrow-5.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:4d8adda1892ef4553c4804af7f67cce484f4d6371564e2d8374b8e2bc85293e2"}, + {file = "pyarrow-5.0.0.tar.gz", hash = "sha256:24e64ea33eed07441cc0e80c949e3a1b48211a1add8953268391d250f4d39922"}, +] +pycodestyle = [ + {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, + {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, +] +pyflakes = [ + {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, + {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, +] +pyparsing = [ + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, +] +pytest = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +pytz = [ + {file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"}, + {file = "pytz-2021.1.tar.gz", hash = "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da"}, +] +pyyaml = [ + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, +] +regex = [ + {file = "regex-2021.8.28-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d05ad5367c90814099000442b2125535e9d77581855b9bee8780f1b41f2b1a2"}, + {file = "regex-2021.8.28-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3bf1bc02bc421047bfec3343729c4bbbea42605bcfd6d6bfe2c07ade8b12d2a"}, + {file = "regex-2021.8.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f6a808044faae658f546dd5f525e921de9fa409de7a5570865467f03a626fc0"}, + {file = "regex-2021.8.28-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a617593aeacc7a691cc4af4a4410031654f2909053bd8c8e7db837f179a630eb"}, + {file = "regex-2021.8.28-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:79aef6b5cd41feff359acaf98e040844613ff5298d0d19c455b3d9ae0bc8c35a"}, + {file = "regex-2021.8.28-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0fc1f8f06977c2d4f5e3d3f0d4a08089be783973fc6b6e278bde01f0544ff308"}, + {file = "regex-2021.8.28-cp310-cp310-win32.whl", hash = "sha256:6eebf512aa90751d5ef6a7c2ac9d60113f32e86e5687326a50d7686e309f66ed"}, + {file = "regex-2021.8.28-cp310-cp310-win_amd64.whl", hash = "sha256:ac88856a8cbccfc14f1b2d0b829af354cc1743cb375e7f04251ae73b2af6adf8"}, + {file = "regex-2021.8.28-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c206587c83e795d417ed3adc8453a791f6d36b67c81416676cad053b4104152c"}, + {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8690ed94481f219a7a967c118abaf71ccc440f69acd583cab721b90eeedb77c"}, + {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:328a1fad67445550b982caa2a2a850da5989fd6595e858f02d04636e7f8b0b13"}, + {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c7cb4c512d2d3b0870e00fbbac2f291d4b4bf2634d59a31176a87afe2777c6f0"}, + {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66256b6391c057305e5ae9209941ef63c33a476b73772ca967d4a2df70520ec1"}, + {file = "regex-2021.8.28-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8e44769068d33e0ea6ccdf4b84d80c5afffe5207aa4d1881a629cf0ef3ec398f"}, + {file = "regex-2021.8.28-cp36-cp36m-win32.whl", hash = "sha256:08d74bfaa4c7731b8dac0a992c63673a2782758f7cfad34cf9c1b9184f911354"}, + {file = "regex-2021.8.28-cp36-cp36m-win_amd64.whl", hash = "sha256:abb48494d88e8a82601af905143e0de838c776c1241d92021e9256d5515b3645"}, + {file = "regex-2021.8.28-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b4c220a1fe0d2c622493b0a1fd48f8f991998fb447d3cd368033a4b86cf1127a"}, + {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a332404baa6665b54e5d283b4262f41f2103c255897084ec8f5487ce7b9e8e"}, + {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c61dcc1cf9fd165127a2853e2c31eb4fb961a4f26b394ac9fe5669c7a6592892"}, + {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ee329d0387b5b41a5dddbb6243a21cb7896587a651bebb957e2d2bb8b63c0791"}, + {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f60667673ff9c249709160529ab39667d1ae9fd38634e006bec95611f632e759"}, + {file = "regex-2021.8.28-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b844fb09bd9936ed158ff9df0ab601e2045b316b17aa8b931857365ea8586906"}, + {file = "regex-2021.8.28-cp37-cp37m-win32.whl", hash = "sha256:4cde065ab33bcaab774d84096fae266d9301d1a2f5519d7bd58fc55274afbf7a"}, + {file = "regex-2021.8.28-cp37-cp37m-win_amd64.whl", hash = "sha256:1413b5022ed6ac0d504ba425ef02549a57d0f4276de58e3ab7e82437892704fc"}, + {file = "regex-2021.8.28-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ed4b50355b066796dacdd1cf538f2ce57275d001838f9b132fab80b75e8c84dd"}, + {file = "regex-2021.8.28-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28fc475f560d8f67cc8767b94db4c9440210f6958495aeae70fac8faec631797"}, + {file = "regex-2021.8.28-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdc178caebd0f338d57ae445ef8e9b737ddf8fbc3ea187603f65aec5b041248f"}, + {file = "regex-2021.8.28-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999ad08220467b6ad4bd3dd34e65329dd5d0df9b31e47106105e407954965256"}, + {file = "regex-2021.8.28-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:808ee5834e06f57978da3e003ad9d6292de69d2bf6263662a1a8ae30788e080b"}, + {file = "regex-2021.8.28-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d5111d4c843d80202e62b4fdbb4920db1dcee4f9366d6b03294f45ed7b18b42e"}, + {file = "regex-2021.8.28-cp38-cp38-win32.whl", hash = "sha256:473858730ef6d6ff7f7d5f19452184cd0caa062a20047f6d6f3e135a4648865d"}, + {file = "regex-2021.8.28-cp38-cp38-win_amd64.whl", hash = "sha256:31a99a4796bf5aefc8351e98507b09e1b09115574f7c9dbb9cf2111f7220d2e2"}, + {file = "regex-2021.8.28-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:04f6b9749e335bb0d2f68c707f23bb1773c3fb6ecd10edf0f04df12a8920d468"}, + {file = "regex-2021.8.28-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b006628fe43aa69259ec04ca258d88ed19b64791693df59c422b607b6ece8bb"}, + {file = "regex-2021.8.28-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:121f4b3185feaade3f85f70294aef3f777199e9b5c0c0245c774ae884b110a2d"}, + {file = "regex-2021.8.28-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a577a21de2ef8059b58f79ff76a4da81c45a75fe0bfb09bc8b7bb4293fa18983"}, + {file = "regex-2021.8.28-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1743345e30917e8c574f273f51679c294effba6ad372db1967852f12c76759d8"}, + {file = "regex-2021.8.28-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e1e8406b895aba6caa63d9fd1b6b1700d7e4825f78ccb1e5260551d168db38ed"}, + {file = "regex-2021.8.28-cp39-cp39-win32.whl", hash = "sha256:ed283ab3a01d8b53de3a05bfdf4473ae24e43caee7dcb5584e86f3f3e5ab4374"}, + {file = "regex-2021.8.28-cp39-cp39-win_amd64.whl", hash = "sha256:610b690b406653c84b7cb6091facb3033500ee81089867ee7d59e675f9ca2b73"}, + {file = "regex-2021.8.28.tar.gz", hash = "sha256:f585cbbeecb35f35609edccb95efd95a3e35824cd7752b586503f7e6087303f1"}, +] +requests = [ + {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, + {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, +] +sacremoses = [ + {file = "sacremoses-0.0.45-py3-none-any.whl", hash = "sha256:fa93db44bc04542553ba6090818b892f603d02aa0d681e6c5c3023baf17e8564"}, + {file = "sacremoses-0.0.45.tar.gz", hash = "sha256:58176cc28391830789b763641d0f458819bebe88681dac72b41a19c0aedc07e9"}, +] +sentry-sdk = [ + {file = "sentry-sdk-1.3.1.tar.gz", hash = "sha256:ebe99144fa9618d4b0e7617e7929b75acd905d258c3c779edcd34c0adfffe26c"}, + {file = "sentry_sdk-1.3.1-py2.py3-none-any.whl", hash = "sha256:f33d34c886d0ba24c75ea8885a8b3a172358853c7cbde05979fc99c29ef7bc52"}, +] +shortuuid = [ + {file = "shortuuid-1.0.1-py3-none-any.whl", hash = "sha256:492c7402ff91beb1342a5898bd61ea953985bf24a41cd9f247409aa2e03c8f77"}, + {file = "shortuuid-1.0.1.tar.gz", hash = "sha256:3c11d2007b915c43bee3e10625f068d8a349e04f0d81f08f5fa08507427ebf1f"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +smmap = [ + {file = "smmap-4.0.0-py2.py3-none-any.whl", hash = "sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2"}, + {file = "smmap-4.0.0.tar.gz", hash = "sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182"}, +] +subprocess32 = [ + {file = "subprocess32-3.5.4-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:88e37c1aac5388df41cc8a8456bb49ebffd321a3ad4d70358e3518176de3a56b"}, + {file = "subprocess32-3.5.4-cp27-cp27mu-manylinux2014_x86_64.whl", hash = "sha256:e45d985aef903c5b7444d34350b05da91a9e0ea015415ab45a21212786c649d0"}, + {file = "subprocess32-3.5.4.tar.gz", hash = "sha256:eb2937c80497978d181efa1b839ec2d9622cf9600a039a79d0e108d1f9aec79d"}, +] +tokenizers = [ + {file = "tokenizers-0.10.3-cp36-cp36m-macosx_10_11_x86_64.whl", hash = "sha256:4ab688daf4692a6c31dfe42f1f3a4a8c22050705eb69d58d3efde9d55f434586"}, + {file = "tokenizers-0.10.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c26dbc3b2a3d71d3d40c50975ec62145932f05aea73f03ea35c48ebd3a717611"}, + {file = "tokenizers-0.10.3-cp36-cp36m-win32.whl", hash = "sha256:6b84673997990b3c260ae2f7c57fdf1f835e316820eff14aca46dc68be3c0c74"}, + {file = "tokenizers-0.10.3-cp36-cp36m-win_amd64.whl", hash = "sha256:2a9ee3ee574d4aa740e099b0ad6ef8e63f52f48cde359bb31801146a5aa614dc"}, + {file = "tokenizers-0.10.3-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:2f8c5fefef0d0a03be613547e613fbda06b9e6ee0891236649524964c3e54d80"}, + {file = "tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4cc194104c8e427ffc4f54c7866488b42f2b1f6351a6cad0d045ca5ab8108e42"}, + {file = "tokenizers-0.10.3-cp37-cp37m-win32.whl", hash = "sha256:edd8cb85c16b4b65e87ea5ef9d400be9fdd53c4152adbaca8817e16dd3aa480b"}, + {file = "tokenizers-0.10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:7b11b373705d082d43657c08883b79b5330f1952f0668d17488b6b889c4d7feb"}, + {file = "tokenizers-0.10.3-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:a7ce0c2f27f7c92aa3f895231de90319acdf960ce2e42ba591edc651fda7d3c9"}, + {file = "tokenizers-0.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ae7e40d9c8a77c5a4109731ac3e21633b0c609c56a8b58be6b863da61fa54636"}, + {file = "tokenizers-0.10.3-cp38-cp38-win32.whl", hash = "sha256:a7ce051aafc53c564c9edbc09df300c2bd4f6ce87460fc22a276fed405d1892a"}, + {file = "tokenizers-0.10.3-cp38-cp38-win_amd64.whl", hash = "sha256:91a8c045980594c7c437a52c3da5276eb3c530a662b4ef628ff32d81fb22b543"}, + {file = "tokenizers-0.10.3-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:1d8867db210d75d97312360ae23b92aeb6a6b5bc65e15c1cd9d204b3fa3fc262"}, + {file = "tokenizers-0.10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:18c495e700f4588b9a00e58b4c41dc459c36daaa7c39a27faf880eb8f5533ce1"}, + {file = "tokenizers-0.10.3-cp39-cp39-win32.whl", hash = "sha256:ad700fd9da518884fd58bf89f0b6dfeecef9b4e2d2db8765ef259f66d6c14980"}, + {file = "tokenizers-0.10.3-cp39-cp39-win_amd64.whl", hash = "sha256:e9d147e545cdfeca560646c7a703bf287afe45645da426506ccd5eb78aab5ef5"}, + {file = "tokenizers-0.10.3.tar.gz", hash = "sha256:1a5d3b596c6d3a237e1ad7f46c472d467b0246be7fd1a364f12576eb8db8f7e6"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +tomli = [ + {file = "tomli-1.2.1-py3-none-any.whl", hash = "sha256:8dd0e9524d6f386271a36b41dbf6c57d8e32fd96fd22b6584679dc569d20899f"}, + {file = "tomli-1.2.1.tar.gz", hash = "sha256:a5b75cb6f3968abb47af1b40c1819dc519ea82bcc065776a866e8d74c5ca9442"}, +] +torch = [ + {file = "torch-1.8.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:f23eeb1a48cc39209d986c418ad7e02227eee973da45c0c42d36b1aec72f4940"}, + {file = "torch-1.8.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:4ace9c5bb94d5a7b9582cd089993201658466e9c59ff88bd4e9e08f6f072d1cf"}, + {file = "torch-1.8.1-cp36-cp36m-win_amd64.whl", hash = "sha256:6ffa1e7ae079c7cb828712cb0cdaae5cc4fb87c16a607e6d14526b62c20bcc17"}, + {file = "torch-1.8.1-cp36-none-macosx_10_9_x86_64.whl", hash = "sha256:16f2630d9604c4ee28ea7d6e388e2264cd7bc6031c6ecd796bae3f56b5efa9a3"}, + {file = "torch-1.8.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:95b7bbbacc3f28fe438f418392ceeae146a01adc03b29d44917d55214ac234c9"}, + {file = "torch-1.8.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:55137feb2f5a0dc7aced5bba690dcdb7652054ad3452b09a2bbb59f02a11e9ff"}, + {file = "torch-1.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:8ad2252bf09833dcf46a536a78544e349b8256a370e03a98627ebfb118d9555b"}, + {file = "torch-1.8.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:1388b30fbd262c1a053d6c9ace73bb0bd8f5871b4892b6f3e02d1d7bc9768563"}, + {file = "torch-1.8.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:e7ad1649adb7dc2a450e70a3e51240b84fa4746c69c8f98989ce0c254f9fba3a"}, + {file = "torch-1.8.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3e4190c04dfd89c59bad06d5fe451446643a65e6d2607cc989eb1001ee76e12f"}, + {file = "torch-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:5c2e9a33d44cdb93ebd739b127ffd7da786bf5f740539539195195b186a05f6c"}, + {file = "torch-1.8.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:c6ede2ae4dcd8214b63e047efabafa92493605205a947574cf358216ca4e440a"}, + {file = "torch-1.8.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:ce7d435426f3dd14f95710d779aa46e9cd5e077d512488e813f7589fdc024f78"}, + {file = "torch-1.8.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:a50ea8ed900927fb30cadb63aa7a32fdd59c7d7abe5012348dfbe35a8355c083"}, + {file = "torch-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:dac4d10494e74f7e553c92d7263e19ea501742c4825ddd26c4decfa27be95981"}, + {file = "torch-1.8.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:225ee4238c019b28369c71977327deeeb2bd1c6b8557e6fcf631b8866bdc5447"}, +] +tqdm = [ + {file = "tqdm-4.62.2-py2.py3-none-any.whl", hash = "sha256:80aead664e6c1672c4ae20dc50e1cdc5e20eeff9b14aa23ecd426375b28be588"}, + {file = "tqdm-4.62.2.tar.gz", hash = "sha256:a4d6d112e507ef98513ac119ead1159d286deab17dffedd96921412c2d236ff5"}, +] +transformers = [ + {file = "transformers-4.10.0-py3-none-any.whl", hash = "sha256:2a27281be8463b16e23692e960cbd14825b270ca939cb01957ed19136708790f"}, + {file = "transformers-4.10.0.tar.gz", hash = "sha256:04b4ec9cfc0ed6e6881c1ce3c79f738069a78df8a8b528a0244a686b6c129601"}, +] +typed-ast = [ + {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, + {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, + {file = "typed_ast-1.4.3-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528"}, + {file = "typed_ast-1.4.3-cp35-cp35m-win32.whl", hash = "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428"}, + {file = "typed_ast-1.4.3-cp35-cp35m-win_amd64.whl", hash = "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3"}, + {file = "typed_ast-1.4.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f"}, + {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341"}, + {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace"}, + {file = "typed_ast-1.4.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f"}, + {file = "typed_ast-1.4.3-cp36-cp36m-win32.whl", hash = "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363"}, + {file = "typed_ast-1.4.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7"}, + {file = "typed_ast-1.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266"}, + {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e"}, + {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04"}, + {file = "typed_ast-1.4.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899"}, + {file = "typed_ast-1.4.3-cp37-cp37m-win32.whl", hash = "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c"}, + {file = "typed_ast-1.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805"}, + {file = "typed_ast-1.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a"}, + {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff"}, + {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41"}, + {file = "typed_ast-1.4.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39"}, + {file = "typed_ast-1.4.3-cp38-cp38-win32.whl", hash = "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927"}, + {file = "typed_ast-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40"}, + {file = "typed_ast-1.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3"}, + {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4"}, + {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0"}, + {file = "typed_ast-1.4.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3"}, + {file = "typed_ast-1.4.3-cp39-cp39-win32.whl", hash = "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808"}, + {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, + {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, +] +typing-extensions = [ + {file = "typing_extensions-3.10.0.2-py2-none-any.whl", hash = "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7"}, + {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, + {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, +] +urllib3 = [ + {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, + {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, +] +wandb = [ + {file = "wandb-0.12.1-py2.py3-none-any.whl", hash = "sha256:723e81cca072def483422b9ae45d5c1b5c8f2d011dc5c85c8891696f8504a55e"}, + {file = "wandb-0.12.1.tar.gz", hash = "sha256:739f205b5c9a5bc25e742f1c707e65322cd6aafba8ec64299e5757f992f34be3"}, +] +xxhash = [ + {file = "xxhash-2.0.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:dac3b94881b943bbe418f5829128b9c48f69a66f816ef8b72ee0129d676dbd7c"}, + {file = "xxhash-2.0.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:43fd97f332bd581639bb99fe8f09f7e9113d49cad4d21bef0620867f92c802c6"}, + {file = "xxhash-2.0.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:6e5058c3fa5b42ded9a303f1a5a42d3ff732cb54c108424c63e993fc3379513c"}, + {file = "xxhash-2.0.2-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:dfacce97a3ccb46089e358ceaeca9300298511673bf87596da66882af386f6c7"}, + {file = "xxhash-2.0.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:1dfa115c8e07b3e1d94ebd60a6d6ee16ea692efb890e245addb0d33b47ee1dee"}, + {file = "xxhash-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:fb28b0313c7582225373f343635674231518452331a9bdea8261d0e27b48594f"}, + {file = "xxhash-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:427851234a87bfe6636c90b89bd65b7ca913befff3c7bcd92a3568e635fccc92"}, + {file = "xxhash-2.0.2-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:0b92a01dc8dcada8827de140a5df83c9e8e5c190ef8bf972c98ebbe0924ee044"}, + {file = "xxhash-2.0.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:676d6964b8a9bdaf737ae6836b886ab53b2863c6aa00d43952b130a6130d1bdc"}, + {file = "xxhash-2.0.2-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:8362693a1ce5c1373f48f047470e7797ed17dfe5babc37ba7bef50d6e6f83a72"}, + {file = "xxhash-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:515747159fccd23fc9d1b7afeaa8bd7fc36884188b47491713d22032c5f9e502"}, + {file = "xxhash-2.0.2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:e1787b9cea43f256f8d06c8429999d386a9da9cb000c265a4dde48dd08242528"}, + {file = "xxhash-2.0.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:d47ab1245ee4c7e6fc424ad990e4d7cfe0f206d617efe990fea34000a9242102"}, + {file = "xxhash-2.0.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:81ec049f4936a49311e1fc58036d7d682b5c83d6d16ba1c852a981588c90e027"}, + {file = "xxhash-2.0.2-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:df71aeedee74eaf670d1243b6722c8c77626f3b6e6cf2cd79f2e336b151749cd"}, + {file = "xxhash-2.0.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a922315c8e20dae0d35e54b49fd7ee348fe0a5e2fd8ec02f6a74140e063fcdb3"}, + {file = "xxhash-2.0.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:22ddd484cd92d138feeec556387894b8ec529bab7f2feb3a177eb84baadee8c1"}, + {file = "xxhash-2.0.2-cp35-cp35m-win32.whl", hash = "sha256:b4964e7ddca1ef9d7addef40a9f5eaa97aeda367c1d895e392533c0d2f9c3b8e"}, + {file = "xxhash-2.0.2-cp35-cp35m-win_amd64.whl", hash = "sha256:6077fdb44f68920c4ac8e2f34b2a107c9a218f00a698253c824a0c6c1b9622a3"}, + {file = "xxhash-2.0.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:04ae5706ddfe0fd2b46cd0b6487d3edae7e724e27d732b055ffd0f9539c4afc5"}, + {file = "xxhash-2.0.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c4a892bc47b6ea92bbb82499a81882548ce990d62c1862b3834f1f70e8cf4423"}, + {file = "xxhash-2.0.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:57d43ce9594676b503c0a0a383481cb4e5cf736f88970bd41849fe15a68a5d48"}, + {file = "xxhash-2.0.2-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:c2e44d162c3361392dbde736ee8ba3d1a414f63e32be6c71186f2b0654559d26"}, + {file = "xxhash-2.0.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:0beb79835ca47af257f8126fccd9d5e0ba56ba7d39dab6f6b5a7acea4d8ac4b5"}, + {file = "xxhash-2.0.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:f2bef10c417c4667310cc240d49e521e6b5fc90c4ff77a1ec78649869685e8d3"}, + {file = "xxhash-2.0.2-cp36-cp36m-win32.whl", hash = "sha256:9b6bb1bd34a6365c790c328a604ec5a628059fef6e4486380caa89bc12787a6e"}, + {file = "xxhash-2.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:4243dbeb1ce09d359289844f0c54676343857fdc6a092184aea159fecdf6d9f3"}, + {file = "xxhash-2.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:71b38300e1803ab32ee787f89cdbc032b46ac5834eca9109d8fb576ae1a31741"}, + {file = "xxhash-2.0.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:a8a68d117178f15c96cb9ae2613f53db94e0fdb34ffc69c7ab600c899c7a966c"}, + {file = "xxhash-2.0.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:dd9c72520f790ce6eaa535cdad1a53ded22deab43766cfa7cef42834a9a65561"}, + {file = "xxhash-2.0.2-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:f95adf6091fa13ce19fab21fadb8d07210822320568d24a6405d6b557afc0411"}, + {file = "xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:00aaf882036d2a0fa7652cf9aeaaf2ad077b784c09ef8d60f5d97ebf0d47ffa1"}, + {file = "xxhash-2.0.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:bb8c0efad20da40da1aa56f36b929b965d1adede8a1d5b37b702d378a683e0dd"}, + {file = "xxhash-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:6fc0b8c21a181b771e1f0c25eb8a0a241af0126f1fc19f4c3cde7233de91326f"}, + {file = "xxhash-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b232b47a3aa825e0df14b1bd3e051dd327c8539e382728ddb81997d26de5256a"}, + {file = "xxhash-2.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc328d3d635ec851d6befdf6ced2134d587d3be973dbbbc489da24c0c88ecb01"}, + {file = "xxhash-2.0.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9e6e5e095417060bed45119c510d5bc846b62e2a8218cb3e5a19b3ccf12e4c18"}, + {file = "xxhash-2.0.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:b4b7d4d19c125738c5fc48356505dfbd63b3cdf826dd868a1b80a73de48729b7"}, + {file = "xxhash-2.0.2-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:686fcf2aff041df65470eccc7dcea5e7e77cfad99efcaba0c6f58bbd81846e10"}, + {file = "xxhash-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:cb3a196fd1d55ce86b1123cbf3ef6603f80f4d0b46541412bb5056b0563ef384"}, + {file = "xxhash-2.0.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:68d067427f2c6f7b3014e28bf4794b0876ab5f6366b53e1d6f59d275b4f19a8d"}, + {file = "xxhash-2.0.2-cp38-cp38-win32.whl", hash = "sha256:73649555656dd17e809b9b3c54855f4f72144024b0e6395cd37b5395fa0f48c3"}, + {file = "xxhash-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:dafd1066c99d448a7a1226f10766b61ff752aaad8a4392e4cae30aafefa6fff5"}, + {file = "xxhash-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eb1e9e347c9810a272154814cf5ce33a6c3ac7d0d7cbcb066e92dd5f9fa4db8f"}, + {file = "xxhash-2.0.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:ebff22f1783f641c6c2b313bfc44d6cc620c17409ec512e67c7c6de809155880"}, + {file = "xxhash-2.0.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:b7640e043ac6e0f503eadb108e6971d69b0c95c23fbcac3e5632578f9f906050"}, + {file = "xxhash-2.0.2-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:db2352d375e6594620c462c029d3c1a1b18ff7168e470657e354f1b8b332d9dd"}, + {file = "xxhash-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f49dbd3b8e4cc13f2df92fb3db39204e3258105a212e23784cbb340e415ae8ed"}, + {file = "xxhash-2.0.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:e70059c5cc8f0cecd16d8cb0263de8f317239cabee3fa4af35c0a1ddaed2110e"}, + {file = "xxhash-2.0.2-cp39-cp39-win32.whl", hash = "sha256:a0199a07a264be96ed658ba3b4e9ee58a3c678e51a18e134e2518cf1a8171e18"}, + {file = "xxhash-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:173d3f662dc88de734bd622e46a3bbac6fd00e957b3e098fa8b75b141aa4354e"}, + {file = "xxhash-2.0.2-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:e94fdff9b102ca7c0969230d209f7ce17020db17a89d026ac45d8ffb9e4929ec"}, + {file = "xxhash-2.0.2-pp27-pypy_73-manylinux1_x86_64.whl", hash = "sha256:d7175cd7f490aae742d18eb9b519e74180958f88fa8ff47091727b3efb57bfbf"}, + {file = "xxhash-2.0.2-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:d707d2a053a5d55ccd2e59d7a228636cafeebb44c9ac3ca1c088f4d384c8c3a9"}, + {file = "xxhash-2.0.2-pp27-pypy_73-win32.whl", hash = "sha256:dad190caa293abbb39d96b4a09f121fc971d81eb19c96e4e0db89a99a7d59b93"}, + {file = "xxhash-2.0.2-pp36-pypy36_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5dc3da5fa855dd8e35f24d20fabfcd29c0b3ac85a14dc2c329c029971ae4eeb7"}, + {file = "xxhash-2.0.2-pp36-pypy36_pp73-manylinux1_x86_64.whl", hash = "sha256:17a3b0a2ff20879ed5c9d9c178349e9c6257db11b193e4103282d7a78ef9cb08"}, + {file = "xxhash-2.0.2-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:c75f8375c80c3815f49a744ef1a8303577757eb9a2dc53bed33d9318b760fec6"}, + {file = "xxhash-2.0.2-pp36-pypy36_pp73-win32.whl", hash = "sha256:eb2670ed6c435189aeb479bfff990e00b849ae0ff49945632db74b2a2a08d192"}, + {file = "xxhash-2.0.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ff518ec1bd7cc33218f8f3325848c56e9c73c5df30138a64a89dd65ab1e1ffb5"}, + {file = "xxhash-2.0.2-pp37-pypy37_pp73-manylinux1_x86_64.whl", hash = "sha256:c4a0806ffb33c9d892b5565fa010c252c7e0f4d01ded901a637dfede624e4d0c"}, + {file = "xxhash-2.0.2-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:fdfac2014301da79cebcd8f9535c875f63242fe404d741cec5f70f400cc6a561"}, + {file = "xxhash-2.0.2-pp37-pypy37_pp73-win32.whl", hash = "sha256:357f6a52bd18a80635cf4c83f648c42fa0609713b4183929ed019f7627af4b68"}, + {file = "xxhash-2.0.2.tar.gz", hash = "sha256:b7bead8cf6210eadf9cecf356e17af794f57c0939a3d420a00d87ea652f87b49"}, +] +yarl = [ + {file = "yarl-1.6.3-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434"}, + {file = "yarl-1.6.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478"}, + {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6"}, + {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e"}, + {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406"}, + {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76"}, + {file = "yarl-1.6.3-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366"}, + {file = "yarl-1.6.3-cp36-cp36m-win32.whl", hash = "sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721"}, + {file = "yarl-1.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643"}, + {file = "yarl-1.6.3-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e"}, + {file = "yarl-1.6.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3"}, + {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8"}, + {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a"}, + {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c"}, + {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f"}, + {file = "yarl-1.6.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970"}, + {file = "yarl-1.6.3-cp37-cp37m-win32.whl", hash = "sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e"}, + {file = "yarl-1.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50"}, + {file = "yarl-1.6.3-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2"}, + {file = "yarl-1.6.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec"}, + {file = "yarl-1.6.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71"}, + {file = "yarl-1.6.3-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc"}, + {file = "yarl-1.6.3-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959"}, + {file = "yarl-1.6.3-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2"}, + {file = "yarl-1.6.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2"}, + {file = "yarl-1.6.3-cp38-cp38-win32.whl", hash = "sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896"}, + {file = "yarl-1.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a"}, + {file = "yarl-1.6.3-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e"}, + {file = "yarl-1.6.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724"}, + {file = "yarl-1.6.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c"}, + {file = "yarl-1.6.3-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25"}, + {file = "yarl-1.6.3-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96"}, + {file = "yarl-1.6.3-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0"}, + {file = "yarl-1.6.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4"}, + {file = "yarl-1.6.3-cp39-cp39-win32.whl", hash = "sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424"}, + {file = "yarl-1.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6"}, + {file = "yarl-1.6.3.tar.gz", hash = "sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10"}, +] +zipp = [ + {file = "zipp-3.5.0-py3-none-any.whl", hash = "sha256:957cfda87797e389580cb8b9e3870841ca991e2125350677b2ca83a0e99390a3"}, + {file = "zipp-3.5.0.tar.gz", hash = "sha256:f5812b1e007e48cff63449a5e9f4e7ebea716b4111f9c4f9a645f91d579bf0c4"}, +] diff --git a/pyproject.toml b/pyproject.toml index 9cb022a7..d37377c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,28 @@ [tool.black] line-length = 119 -target-version = ['py35'] \ No newline at end of file +target-version = ['py37'] + +[tool.poetry] +name = "metadata" +version = "0.1.0" +description = "Codebase for including metadata (e.g., URLs, timestamps, HTML tags) during language model pretraining." +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = ">=3.7.11, <3.10" +torch = "1.8.1" # 1.9 also works for Colab TPU, but not for GPU/CPU; the bugfix of 1.8.2 seems irrelevant for now. +hydra-core = ">=1.1,<1.2" # Poetry will likely update it to 1.1.1, but it is probably ok and good for bugfixes. +wandb = ">=0.10.32,<1" # Poetry will likely update it to 0.12.1, but it is probably ok and good for bugfixes. +transformers = "^4.6.0" # Poetry will likely update it to 4.10.0, but it is probably ok and good for bugfixes. +accelerate = ">=0.4.0,<1" # We may want to use 0.5.0 in the near future +datasets = {extras = ["streaming"], version = "^1.11.0"} + +[tool.poetry.dev-dependencies] +isort = "^5.5.4" +black = "^21.4b0" +flake8 = "^3.8.3" +pytest = "^6.2.4" + +[build-system] +requires = ["poetry-core>=1.0.0", "setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index 8c6321ee..62be06ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -torch==1.8.1 -hydra_core==1.0.6 -wandb==0.10.30 -transformers==4.6.0 -accelerate==0.3.0 -datasets==1.11.0 +torch==1.8.1 # 1.9 also works for Colab TPU, but not for GPU/CPU; the bugfix of 1.8.2 seems irrelevant for now. +hydra_core>=1.1,<1.2 # pip will likely update it to 1.1.1, but it is probably ok and good for bugfixes. +wandb>=0.10.32,<1 # pip will likely update it to 0.12.1, but it is probably ok and good for bugfixes. +transformers>=4.6.0,<5 # pip will likely update it to 4.10.0, but it is probably ok and good for bugfixes. +accelerate>=0.4.0,<1 # We may want to use 0.5.0 in the near future +datasets[streaming]>=1.11.0,<2 diff --git a/setup.py b/setup.py index 65bf2dea..f71c6605 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,8 @@ def req_file(filename): setup( name="bsmetadata", - version="0.0.0", + python_requires=">=3.7.11, <3.10", + version="0.1.0", url="https://github.com/bigscience-workshop/metadata.git", author="Multiple Authors", author_email="xxx", From c67eb684a98ade30c4a21080357013717ae9826c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=20=E5=A4=A9=E6=88=A9=20Mike=20Tian-Jian=20Jiang?= Date: Mon, 6 Sep 2021 23:48:48 +0900 Subject: [PATCH 2/3] refactor: set position of tqdm for multithread eval without newline (#3) --- bsmetadata/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bsmetadata/train.py b/bsmetadata/train.py index a5d37049..71619927 100644 --- a/bsmetadata/train.py +++ b/bsmetadata/train.py @@ -111,7 +111,7 @@ def main(args: CFG) -> None: set_seed(args.seed) accelerator = Accelerator() is_local_main_process = accelerator.is_local_main_process - tqdm = partial(original_tqdm, disable=not is_local_main_process) + tqdm = partial(original_tqdm, disable=not is_local_main_process, position=0) os.makedirs(args.out_dir, exist_ok=True) From 72419af8971fc4a154ad37ece2759af6e2dd772f Mon Sep 17 00:00:00 2001 From: Mike Tian-Jian Jiang Date: Fri, 3 Sep 2021 22:50:17 +0900 Subject: [PATCH 3/3] feat: add a Colab notebook as TPU playground with basic EDA --- playground.ipynb | 1702 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1702 insertions(+) create mode 100644 playground.ipynb diff --git a/playground.ipynb b/playground.ipynb new file mode 100644 index 00000000..9122aab1 --- /dev/null +++ b/playground.ipynb @@ -0,0 +1,1702 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "BigScience Metadata playground", + "provenance": [], + "collapsed_sections": [ + "SbhiOxCY8rZ2", + "veqze27o9HEX", + "VpEg7z7CAiUc", + "L6_jjYRf0ZXN", + "UI_pyX7P0cRG", + "ckWrWh2r07ZH", + "wexEuNAuOTrI", + "cEPExQdIbqN1", + "6lisgI0hbLUd", + "QluA_8eLbTqr", + "__lnPt_ya_SS", + "0nXMJuiMbcBx", + "j3EMRvoAbf9Y" + ] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "TPU" + }, + "cells": [ + { + "cell_type": "code", + "metadata": { + "id": "rE6KUpKC-xb7" + }, + "source": [ + "# !nvidia-smi -L" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "5DtYDK2LL6jM" + }, + "source": [ + "# import os\n", + "# from tensorflow.python.profiler import profiler_client\n", + "\n", + "# tpu_profile_service_address = os.environ['COLAB_TPU_ADDR'].replace('8470', '8466')\n", + "# print(profiler_client.monitor(tpu_profile_service_address, 100, 2))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "V5QKUdkUMBAr" + }, + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "reQ0s7ej6tO2" + }, + "source": [ + "# Data" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "qX28VK4UYGRX" + }, + "source": [ + "from pathlib import Path\n", + "\n", + "\n", + "DATA_DIR=Path(\"/content/drive/MyDrive/colab_data/bigscience\")\n", + "!mkdir -p {DATA_DIR}\n", + "\n", + "!rm -rf sample_data" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "RwcpNOdzTo-o" + }, + "source": [ + "TRN_CNT, TRN_OFFSET = 10000, 0\n", + "VLD_CNT, VLD_OFFSET = 1000, 1000\n", + "ds_key, ds_subkey = \"mc4\", \"en\"\n", + "ds_name = f\"{ds_key}-{ds_subkey}\"\n", + "trn_jsonl_p = DATA_DIR / f\"{ds_name}_trn_{TRN_OFFSET}-{TRN_OFFSET+TRN_CNT}.jsonl\"\n", + "vld_jsonl_p = DATA_DIR / f\"{ds_name}_vld_{VLD_OFFSET}-{VLD_OFFSET+VLD_CNT}.jsonl\"" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NLyFy7Tr0gJz" + }, + "source": [ + "## URL Metadata of mC4 `(trn[:10000], vld[1000:2000])`\n", + "Skip this subsection if jsonl files of the same data subsets are prepared before." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "oFgAWS0dD2mq" + }, + "source": [ + "!pip check\n", + "!pip install -q datasets[streaming]\n", + "!pip check" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "apuZh6PrVQhW" + }, + "source": [ + "from datasets import load_dataset\n", + "\n", + "\n", + "mc4_en_trn_ds = load_dataset(\n", + " ds_key,\n", + " ds_subkey,\n", + " split=\"train\",\n", + " # data_dir=DATA_DIR,\n", + " streaming=True,\n", + ")\n", + "mc4_en_vld_ds = load_dataset(\n", + " ds_key,\n", + " ds_subkey,\n", + " split=\"validation\",\n", + " # data_dir=DATA_DIR,\n", + " streaming=True,\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "ZstWjsMWygfG" + }, + "source": [ + "!pip check\n", + "!pip install -q jsonlines\n", + "!pip check" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "b4CozMGrjEOu" + }, + "source": [ + "import jsonlines\n", + "\n", + "\n", + "def _gen(ds, limit=None, offset=0):\n", + " for i, data in enumerate(ds):\n", + " if i < offset: continue\n", + " if i - offset == limit: break\n", + " yield {\n", + " \"text\": data['text'],\n", + " \"metadata\": [{\n", + " \"key\": \"url\",\n", + " \"type\": \"global\",\n", + " \"value\": data[\"url\"],\n", + " }],\n", + " }\n", + "\n", + "\n", + "with jsonlines.open(trn_jsonl_p, mode='w') as writer:\n", + " writer.write_all(l for l in _gen(mc4_en_trn_ds, TRN_CNT, TRN_OFFSET))\n", + "with jsonlines.open(vld_jsonl_p, mode='w') as writer:\n", + " writer.write_all(l for l in _gen(mc4_en_vld_ds, VLD_CNT, VLD_OFFSET))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SbhiOxCY8rZ2" + }, + "source": [ + "# Trials" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "veqze27o9HEX" + }, + "source": [ + "## Dependencies" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "qqXXP-0sAzlv" + }, + "source": [ + "!pip check\n", + "!pip install -qU pip\n", + "!pip freeze | grep torch\n", + "!pip check" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "1Fv8-Xr2B21z" + }, + "source": [ + "!pip check\n", + "!pip install -q torch==1.8.1 torchvision==0.9.1 torchtext==0.9.1 -f https://download.pytorch.org/whl/cu102/torch_stable.html\n", + "!pip freeze | grep torch\n", + "!pip check" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VpEg7z7CAiUc" + }, + "source": [ + "### TPU\n", + "Skip this step if not using TPU." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "X_nCrsMuxRYa" + }, + "source": [ + "!pip check\n", + "# From https://github.com/huggingface/notebooks/blob/master/examples/accelerate/simple_nlp_example.ipynb\n", + "# Not sure whether we still need not-yet-released accelerate for TPU or not.\n", + "# !pip install -q cloud-tpu-client==0.10 https://storage.googleapis.com/tpu-pytorch/wheels/torch_xla-1.9-cp37-cp37m-linux_x86_64.whl\n", + "!pip install -q cloud-tpu-client==0.10 https://storage.googleapis.com/tpu-pytorch/wheels/torch_xla-1.8.1-cp37-cp37m-linux_x86_64.whl\n", + "# !pip install -q git+https://github.com/huggingface/accelerate\n", + "!pip check" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fCISNe_6XnZ8" + }, + "source": [ + "### Package" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ptIFglMEAHnG" + }, + "source": [ + "%cd /content\n", + "!rm -rf bigscience-metadata\n", + "!git clone -b perf-collator_with_padding_for_tpu https://github.com/tianjianjiang/bigscience-metadata.git" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "RhnFQ-EXr-JP" + }, + "source": [ + "%cd /content/bigscience-metadata\n", + "!pip install -q -e .\n", + "!pip check" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "4BXNw3yrUX9A" + }, + "source": [ + "!wandb login" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L6_jjYRf0ZXN" + }, + "source": [ + "## Without URL" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "_IdlsR4X0Vfq", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "3fb8ab2b-8df3-401e-c9c2-a023c2fd71bd" + }, + "source": [ + "# !TOKENIZERS_PARALLELISM=false \\\n", + "!accelerate launch --fp16 \\\n", + "/content/bigscience-metadata/bsmetadata/train.py \\\n", + "max_train_steps=100 num_eval=1 \\\n", + "data_config.experiment=without_metadata \\\n", + "data_config.per_device_eval_batch_size=8 \\\n", + "data_config.max_seq_len=768 \\\n", + "data_config.train_file={trn_jsonl_p} \\\n", + "data_config.validation_file={vld_jsonl_p}" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "WARNING:root:TPU has started up successfully with version pytorch-1.8.1\n", + "WARNING:root:TPU has started up successfully with version pytorch-1.8.1\n", + "data_config:\n", + " experiment: without_metadata\n", + " per_device_eval_batch_size: 8\n", + " per_device_train_batch_size: 2\n", + " metadata_list: []\n", + " metadata_sep: ' | '\n", + " metadata_key_value_sep: ': '\n", + " metadata_probability: 1.0\n", + " global_metadata_sep: ' |||'\n", + " max_seq_len: 768\n", + " dataset_name: null\n", + " dataset_config_name: null\n", + " train_file: /content/drive/MyDrive/colab_data/bigscience/mc4-en_trn_0-10000.jsonl\n", + " validation_file: /content/drive/MyDrive/colab_data/bigscience/mc4-en_vld_1000-2000.jsonl\n", + " overwrite_cache: false\n", + " cache_dir: null\n", + " preprocessing_num_workers: null\n", + " validation_split_percentage: 5\n", + " block_size: null\n", + " distributed_type: TPU\n", + "weight_decay: 0.0\n", + "learning_rate: 5.0e-05\n", + "gradient_accumulation_steps: 1\n", + "num_train_epochs: 1\n", + "max_train_steps: 100\n", + "lr_scheduler_type: linear\n", + "num_warmup_steps: 1000\n", + "seed: 42\n", + "out_dir: output_dir\n", + "num_eval: 1\n", + "model_name: gpt2\n", + "project_name: metadata_lm\n", + "\n", + "[2021-09-08 22:35:02,024][datasets.builder][WARNING] - Using custom data configuration default-e08b76da1fb65150\n", + "[2021-09-08 22:35:02,028][datasets.builder][WARNING] - Reusing dataset json (/root/.cache/huggingface/datasets/json/default-e08b76da1fb65150/0.0.0/45636811569ec4a6630521c18235dfbbab83b7ab572e3393c5ba68ccabe98264)\n", + "Running tokenizer on dataset: 100% 10/10 [00:20<00:00, 2.02s/ba]\n", + "Running tokenizer on dataset: 100% 1/1 [00:02<00:00, 2.13s/ba]\n", + "Grouping texts in chunks of 1024: 100% 10/10 [00:36<00:00, 3.60s/ba]\n", + "Grouping texts in chunks of 1024: 100% 1/1 [00:03<00:00, 3.92s/ba]\n", + "training: 0% 0/100 [00:00\n", + " sys.exit(main())\n", + " File \"/usr/local/lib/python3.7/dist-packages/accelerate/commands/accelerate_cli.py\", line 41, in main\n", + " args.func(args)\n", + " File \"/usr/local/lib/python3.7/dist-packages/accelerate/commands/launch.py\", line 384, in launch_command\n", + " simple_launcher(args)\n", + " File \"/usr/local/lib/python3.7/dist-packages/accelerate/commands/launch.py\", line 142, in simple_launcher\n", + " raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)\n", + "subprocess.CalledProcessError: Command '['/usr/bin/python3', '/content/bigscience-metadata/bsmetadata/train.py', 'max_train_steps=100', 'num_eval=1', 'data_config.experiment=without_metadata', 'data_config.per_device_eval_batch_size=8', 'data_config.max_seq_len=768', 'data_config.train_file=/content/drive/MyDrive/colab_data/bigscience/mc4-en_trn_0-10000.jsonl', 'data_config.validation_file=/content/drive/MyDrive/colab_data/bigscience/mc4-en_vld_1000-2000.jsonl']' died with .\n", + "/usr/lib/python3.7/multiprocessing/semaphore_tracker.py:144: UserWarning: semaphore_tracker: There appear to be 6 leaked semaphores to clean up at shutdown\n", + " len(cache))\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ty69LlcHt7ZB" + }, + "source": [ + "# !rm -rf outputs" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UI_pyX7P0cRG" + }, + "source": [ + "## With URL" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "dgPpzdhbnkMd", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "aa02ca4a-5d1e-414a-9564-4cf6f34d68ee" + }, + "source": [ + "# !TOKENIZERS_PARALLELISM=false \\\n", + "!accelerate launch --fp16 \\\n", + "/content/bigscience-metadata/bsmetadata/train.py \\\n", + "max_train_steps=100 num_eval=1 \\\n", + "data_config.experiment=with_metadata \\\n", + "data_config.per_device_eval_batch_size=8 \\\n", + "data_config.max_seq_len=768 \\\n", + "data_config.train_file={trn_jsonl_p} \\\n", + "data_config.validation_file={vld_jsonl_p}" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "WARNING:root:TPU has started up successfully with version pytorch-1.8.1\n", + "WARNING:root:TPU has started up successfully with version pytorch-1.8.1\n", + "data_config:\n", + " experiment: with_metadata\n", + " per_device_eval_batch_size: 8\n", + " per_device_train_batch_size: 2\n", + " metadata_list: []\n", + " metadata_sep: ' | '\n", + " metadata_key_value_sep: ': '\n", + " metadata_probability: 1.0\n", + " global_metadata_sep: ' |||'\n", + " max_seq_len: 768\n", + " dataset_name: null\n", + " dataset_config_name: null\n", + " train_file: /content/drive/MyDrive/colab_data/bigscience/mc4-en_trn_0-10000.jsonl\n", + " validation_file: /content/drive/MyDrive/colab_data/bigscience/mc4-en_vld_1000-2000.jsonl\n", + " overwrite_cache: false\n", + " cache_dir: null\n", + " preprocessing_num_workers: null\n", + " validation_split_percentage: 5\n", + " block_size: null\n", + " distributed_type: TPU\n", + "weight_decay: 0.0\n", + "learning_rate: 5.0e-05\n", + "gradient_accumulation_steps: 1\n", + "num_train_epochs: 1\n", + "max_train_steps: 100\n", + "lr_scheduler_type: linear\n", + "num_warmup_steps: 1000\n", + "seed: 42\n", + "out_dir: output_dir\n", + "num_eval: 1\n", + "model_name: gpt2\n", + "project_name: metadata_lm\n", + "\n", + "[2021-09-08 22:39:13,291][datasets.builder][WARNING] - Using custom data configuration default-e08b76da1fb65150\n", + "[2021-09-08 22:39:13,295][datasets.builder][WARNING] - Reusing dataset json (/root/.cache/huggingface/datasets/json/default-e08b76da1fb65150/0.0.0/45636811569ec4a6630521c18235dfbbab83b7ab572e3393c5ba68ccabe98264)\n", + "Pre-process the text and metadata to create new samples: 100% 10/10 [00:58<00:00, 5.88s/ba]\n", + "Pre-process the text and metadata to create new samples: 100% 1/1 [00:06<00:00, 6.10s/ba]\n", + "Create labels column: 100% 13/13 [00:12<00:00, 1.06ba/s]\n", + "Create labels column: 100% 2/2 [00:01<00:00, 1.60ba/s]\n", + "training: 0% 0/100 [00:00\n", + " sys.exit(main())\n", + " File \"/usr/local/lib/python3.7/dist-packages/accelerate/commands/accelerate_cli.py\", line 41, in main\n", + " args.func(args)\n", + " File \"/usr/local/lib/python3.7/dist-packages/accelerate/commands/launch.py\", line 384, in launch_command\n", + " simple_launcher(args)\n", + " File \"/usr/local/lib/python3.7/dist-packages/accelerate/commands/launch.py\", line 142, in simple_launcher\n", + " raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)\n", + "subprocess.CalledProcessError: Command '['/usr/bin/python3', '/content/bigscience-metadata/bsmetadata/train.py', 'max_train_steps=100', 'num_eval=1', 'data_config.experiment=with_metadata', 'data_config.per_device_eval_batch_size=8', 'data_config.max_seq_len=768', 'data_config.train_file=/content/drive/MyDrive/colab_data/bigscience/mc4-en_trn_0-10000.jsonl', 'data_config.validation_file=/content/drive/MyDrive/colab_data/bigscience/mc4-en_vld_1000-2000.jsonl']' died with .\n", + "/usr/lib/python3.7/multiprocessing/semaphore_tracker.py:144: UserWarning: semaphore_tracker: There appear to be 6 leaked semaphores to clean up at shutdown\n", + " len(cache))\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "UPTWrf_Tt74I" + }, + "source": [ + "# !rm -rf outputs" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ckWrWh2r07ZH" + }, + "source": [ + "## TBA" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "xTkK8Ye2B16g" + }, + "source": [ + "!pip install git+https://git@github.com/bigscience-workshop/promptsource.git@main" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "ozeXjIj79zWv" + }, + "source": [ + "# from promptsource.utils import get_dataset\n", + "# tydiqa_pri_ds, failed = get_dataset(ds_key, ds_subkey)\n", + "# tydiqa_pri_ds, failed\n", + "\n", + "ds_key, ds_subkey = \"tydiqa\", \"primary_task\"\n", + "tydiqa_pri_trn_ds = load_dataset(\n", + " ds_key,\n", + " ds_subkey,\n", + " split=\"train\",\n", + " data_dir=DATA_DIR,\n", + " streaming=True\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "8JYUCIzJQDsw" + }, + "source": [ + "from promptsource.templates import TemplateCollection\n", + "template_collection = TemplateCollection()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Kgrmu0xGQUMp" + }, + "source": [ + "tydiqa_pri_tmpls = template_collection.get_dataset(ds_key, ds_subkey)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "o3cBp_wCQ11h" + }, + "source": [ + "tydiqa_pri_tmpls.all_template_names" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "LIQjZWnvSzW5" + }, + "source": [ + "from promptsource.utils import removeHyphen" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "nPQdsLxMQ89g" + }, + "source": [ + "done = False\n", + "for data in tydiqa_pri_trn_ds:\n", + " if data[\"language\"] != \"english\":\n", + " continue\n", + " for tmpl in tydiqa_pri_tmpls.templates.values():\n", + " inst = tmpl.apply(removeHyphen(data))\n", + " if len(inst) == 2:\n", + " data.pop(\"passage_answer_candidates\", None)\n", + " # data.pop(\"plaintext_start_byte\", None)\n", + " display(data)\n", + " prmp, ans = inst\n", + " display(prmp)\n", + " display(ans)\n", + " outcome = {\n", + " \"text\": prmp,\n", + " \"metadata\": [{\n", + " \"key\": \"url\",\n", + " \"type\": \"global\",\n", + " \"value\": data[\"document_url\"],\n", + " }],\n", + " \"answer\": ans,\n", + " }\n", + " print(outcome)\n", + " done = True\n", + " if done:\n", + " break\n", + " if done:\n", + " break" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AzAgDN7G9XFQ" + }, + "source": [ + "# EDA" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eqjnzx-5a2Tf" + }, + "source": [ + "## Preprocessing for mC4 jsonl" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "8PpaITdECoqr" + }, + "source": [ + "from urllib.parse import urlsplit\n", + "\n", + "\n", + "trn_url_parts_map = dict()\n", + "with jsonlines.open(trn_jsonl_p) as trn_jsonl:\n", + " for trn_json in trn_jsonl:\n", + " trn_url = next(filter(lambda x: x[\"key\"] == \"url\", trn_json[\"metadata\"]))['value']\n", + " trn_url_parts = urlsplit(trn_url)\n", + " trn_url_parts_map[trn_url] = trn_url_parts" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "eCB5jZmJFMq-" + }, + "source": [ + "from urllib.parse import unquote_plus\n", + "\n", + "import pandas as pd\n", + "\n", + "\n", + "trn_url_parts_df = pd.DataFrame(\n", + " data=[(\n", + " url,\n", + " parts.netloc,\n", + " sum(1 for _ in filter(None, parts.netloc.split('.'))),\n", + " parts.hostname if parts.hostname else '',\n", + " parts.port if parts.port else '',\n", + " unquote_plus(parts.path),\n", + " sum(1 for _ in filter(None, parts.path.split('/'))),\n", + " unquote_plus(parts.query),\n", + " unquote_plus(parts.fragment)\n", + " ) for url, parts in trn_url_parts_map.items()],\n", + " columns=[\n", + " 'url',\n", + " 'netloc',\n", + " 'netloc_level_cnt',\n", + " 'hostname',\n", + " 'port',\n", + " 'path',\n", + " 'path_level_cnt',\n", + " 'query',\n", + " 'fragment'\n", + " ]\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HG-g6IZqcCdk" + }, + "source": [ + "## Basic Stats" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 326 + }, + "id": "T-xQa8YSQGBW", + "outputId": "db5c0976-0d02-4600-905c-e120de3e7bad" + }, + "source": [ + "trn_url_parts_df.head()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
urlnetlocnetloc_level_cnthostnameportpathpath_level_cntqueryfragment
0http://www.polkaudio.com/forums/showthread.php...www.polkaudio.com3www.polkaudio.com/forums/showthread.php258429-Are-my-speakers-magnetically-shielded&go...
1http://www.atthecoachhouse.co.uk/events/practi...www.atthecoachhouse.co.uk4www.atthecoachhouse.co.uk/events/practical-course-in-the-natural-crafts...2
2https://www.digitalspy.com/showbiz/a26775099/l...www.digitalspy.com3www.digitalspy.com/showbiz/a26775099/louis-tomlinson-zayn-malik-...3
3http://www.californialandcan.org/Plumas/Farm-R...www.californialandcan.org3www.californialandcan.org/Plumas/Farm-Resources/2
4https://www.seattlepi.com/local/article/Report...www.seattlepi.com3www.seattlepi.com/local/article/Report-cites-major-problems-at-...3
\n", + "
" + ], + "text/plain": [ + " url ... fragment\n", + "0 http://www.polkaudio.com/forums/showthread.php... ... \n", + "1 http://www.atthecoachhouse.co.uk/events/practi... ... \n", + "2 https://www.digitalspy.com/showbiz/a26775099/l... ... \n", + "3 http://www.californialandcan.org/Plumas/Farm-R... ... \n", + "4 https://www.seattlepi.com/local/article/Report... ... \n", + "\n", + "[5 rows x 9 columns]" + ] + }, + "metadata": {}, + "execution_count": null + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 359 + }, + "id": "KK7CcZXXT4sR", + "outputId": "0acfd2b0-818d-47ca-df2d-70de695bf132" + }, + "source": [ + "trn_url_parts_df.describe(percentiles=[.25, .5, .75, .95, .975])" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
netloc_level_cntpath_level_cnt
count10000.00000010000.000000
mean2.8457002.495700
std0.5833741.398491
min2.0000000.000000
25%2.0000001.000000
50%3.0000002.000000
75%3.0000003.000000
95%4.0000005.000000
97.5%4.0000006.000000
max7.00000014.000000
\n", + "
" + ], + "text/plain": [ + " netloc_level_cnt path_level_cnt\n", + "count 10000.000000 10000.000000\n", + "mean 2.845700 2.495700\n", + "std 0.583374 1.398491\n", + "min 2.000000 0.000000\n", + "25% 2.000000 1.000000\n", + "50% 3.000000 2.000000\n", + "75% 3.000000 3.000000\n", + "95% 4.000000 5.000000\n", + "97.5% 4.000000 6.000000\n", + "max 7.000000 14.000000" + ] + }, + "metadata": {}, + "execution_count": null + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 282 + }, + "id": "-4ZLCRmOT4sR", + "outputId": "373f2605-6e13-48d3-db1d-be904ba1f533" + }, + "source": [ + "trn_url_parts_df.netloc_level_cnt.value_counts().sort_index().plot()\n", + "trn_url_parts_df.path_level_cnt.value_counts().sort_index().plot()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": null + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xcdZn48c8zmdybNJM2TS+T0kJLL8rVcsfLUi7lokUFBEQKol1ddN39seuCrqKy7LLrBXQVEKFQEAUEXCotlFLAK2pTgQLNhIZyaUrSpE2bS9Nc5/n9cc6k03aSTJLJXM4879crrznznTNnnunlOd98z/c8X1FVjDHGZAdfqgMwxhiTPJb0jTEmi1jSN8aYLGJJ3xhjsoglfWOMySL+VAcwlMmTJ+usWbNSHYYxxmSUjRs37lTVilivpXXSnzVrFtXV1akOwxhjMoqIvDPYaza8Y4wxWcSSvjHGZBFL+sYYk0Us6RtjTBaxpG+MMVnEkr4xxmQRS/rGGJNFLOl72NrXG2lo3ZfqMIwxacSSvkd1dPfxhZ9vZMUf3kp1KMaYNGJJ36NqG9tRhXdbOlMdijEmjVjS96hQYxsA9btteMcYs58lfY8KNbQDsM16+saYKJb0PSrS02/r6qN1X2+KozHGpAtL+h6kqoQa2qkoyQegfrf19o0xDkv6HrR9zz7au/s4c8EUwMb1jTH7WdL3oNpGZzz/zAWVgI3rG2P2s6TvQSE36Z90+CQm5Putp2+MGWBJ34NqGtqYWV7EhHw/wUChjekbYwZY0vegUGM786eWABAMFFlP3xgzwJK+x3T19rO1uSMq6ReyraUTVU1xZMaYdGBJ32PqmjoIK8yfVgpAVXkRe3v62d1pc/WNMZb0PaemwbkpK7qnDzZX3xjjiCvpi0iZiDwqIiERqRGRU0SkXETWicgW9zHg7isi8iMRqRORTSJyfNRxlrn7bxGRZeP1pbJZbWM7Bbk+DptUDEBVoAiAbS02rm+Mib+n/0PgaVWdDxwD1ADXA+tVdS6w3n0OcC4w1/1ZDtwBICLlwI3AScCJwI2RE4VJnFBjO/MqS8jxCQDBcuvpG2P2Gzbpi8hE4EPAPQCq2qOqe4ClwEp3t5XAhe72UuB+dfwZKBORacA5wDpVbVHV3cA6YElCv40h1NjG/KmlA89LC3KZWJjLNkv6xhji6+nPBpqBe0XkJRG5W0SKgUpVbXD3aQQq3e0ZwLao99e7bYO1H0BElotItYhUNzc3j+zbZLnm9m52dvQwf1rJAe3OXH0b3jHGxJf0/cDxwB2qehywl/1DOQCoMx8wIXMCVfUuVV2kqosqKioSccisEamsOW/qgUm/KlBkpRiMMUB8Sb8eqFfVv7jPH8U5Cexwh21wH5vc17cDVVHvD7ptg7WbBInU0I8e3oH9PX2bq2+MGTbpq2ojsE1E5rlNi4HNwCogMgNnGfCEu70KuNKdxXMy0OoOA60FzhaRgHsB92y3zSRITWMblaX5lBfnHdBeVV5Ed1+Y5o7uFEVmjEkX/jj3+zLwoIjkAVuBq3FOGI+IyDXAO8Al7r5rgPOAOqDT3RdVbRGRm4AN7n7fUdWWhHwLAzjTNQ/u5QNUDczg2ceUkoJkh2WMSSNxJX1VfRlYFOOlxTH2VeDaQY6zAlgxkgBNfPr6w2zZ0cHpcycf8lpwYK5+J8fPtFmyxmQzuyPXI97auZee/jALYvT099+VazN4jMl2lvQ9osatoX/wdE2Aojw/k4rz7AYtY4wlfa8INbTh9wmHT54Q8/VguZVYNsZY0veMUGM7c6ZMIM8f+680UmLZGJPdLOl7RKihbaCyZixVgSK279lHOGxz9Y3JZpb0PaC1s5f3WrsGaujHEgwU0tuv7GjvSmJkxph0Y0nfA2p3RO7EHaKnX24llo0xlvQ9IVJzZ8EwPX2wEsvGZDtL+h5Q09BOoCiXKSX5g+4zo8xJ+tbTNya7WdL3gFBjG/OmliAig+5TkJvDlJJ86+kbk+Us6We4cFgHrblzsKryIltMxZgsZ0k/w23b3UlnTz8LYtyJe7AqW0zFmKxnST/D1QxSQz+WYKCIhtYu+vrD4x2WMSZNWdLPcLWN7YjAkZVx9PTLC+kPKw2tNlffmGxlST/DhRrbmD2pmMK8nGH3HSixbOP6xmQtS/oZLtTYHrOyZixVbtK3cX1jspcl/QzW2dPH27v2Mq9y+PF8gGllBfgE6q3wmjFZy5J+BntjRweqsWvox5Kb42PaRJvBY0w2s6SfwUINbvmFOGbuRMwIFNqYvjFZzJJ+Bgs1tlOclzNQVyceVQFbTMWYbGZJP4PVNDjlF3y+wcsvHCwYKKSxrYvuvv5xjMwYk67iSvoi8raIvCoiL4tItdtWLiLrRGSL+xhw20VEfiQidSKySUSOjzrOMnf/LSKybHy+UnZQVWp3tA9ZQz+WqvIiVKFhj83VNyYbjaSn/3eqeqyqLnKfXw+sV9W5wHr3OcC5wFz3ZzlwBzgnCeBG4CTgRODGyInCjNyOtm72dPayYIga+rFEhoJsXN+Y7DSW4Z2lwEp3eyVwYVT7/er4M1AmItOAc4B1qtqiqruBdcCSMXx+Vqtxa+jPG8FFXLDFVIzJdvEmfQWeEZGNIrLcbatU1QZ3uxGodLdnANui3lvvtg3WfgARWS4i1SJS3dzcHGd42Sfk1tyZN8Ke/tTSAvw+sRLLxmQpf5z7na6q20VkCrBORELRL6qqikhCVtxW1buAuwAWLVpkq3gPItTYxoyyQiYW5o7ofTk+YXpZIdtsBo8xWSmunr6qbncfm4Bf44zJ73CHbXAfm9zdtwNVUW8Pum2DtZtRCDW0D7km7lCqygutp29Mlho26YtIsYiURLaBs4HXgFVAZAbOMuAJd3sVcKU7i+dkoNUdBloLnC0iAfcC7tlumxmhnr4wbzZ3xH0n7sGCZUU2pm9MlopneKcS+LW7FJ8f+IWqPi0iG4BHROQa4B3gEnf/NcB5QB3QCVwNoKotInITsMHd7zuq2pKwb5JF3mzuoC+scdXQj6WqvJCdHd109fZTkDt8dU5jjHcMm/RVdStwTIz2XcDiGO0KXDvIsVYAK0YepokWcmfuxLNaVizBgWqbncyZMrpjGGMyk92Rm4FCDe3k+X3MmlQ8qvdXlUfm6tsQjzHZxpJ+BqppbGfulAn4c0b31zfQ07cSy8ZkHUv6GSjU0Dbq8XyAign55Pl9VnjNmCxkST/D7Oropqm9e9Tj+QA+nxAssxLLxmQjS/oZprbRuRN3LD19gGC5lVg2JhtZ0s8wNZGkP4aePjiF17bZmL4xWceSfoapbWxj8oR8Jk/IH9NxqgJF7O7spaO7L0GRGWMygSX9DBNqHH35hWiREstWjsGY7GJJP4P0h5XaBCX9SInleivHYExWsaSfQd7etZfuvvCIV8uKxRZTMSY7WdLPIJEa+ono6U8qzqMwN8cKrxmTZSzpZ5BQYxs5PmHOlAljPpaIWIllY7KQJf0MUtPQzuGTixNWGTMYKLL6O8ZkGUv6GaR2R1tCxvMjqgLW0zcm21jSzxDtXb1sa9mXkPH8iGCgiPauPlo7exN2TGNMerOknyHe2JG4i7gR+0ssW2/fmGxhST9D1ERm7iRweCd6MRVjTHawpJ8hQo1tlBT4mT6xIGHHrBpI+nYx15hsYUk/Q4Qa2lkwtRR3reKEKC30U5Lvt8JrxmQRS/oZQFWdmjtjrKx5MBGxEsvGZBlL+hmgfvc+Orr7xlxDP5ZgwBZTMSabxJ30RSRHRF4SkSfd57NF5C8iUiciD4tIntue7z6vc1+fFXWMG9z2WhE5J9FfxqsiC6fMS+DMnYiqgNPTV9WEH9sYk35G0tP/ClAT9fy/gVtVdQ6wG7jGbb8G2O223+ruh4gsBC4F3gcsAW4XkcTcWupxocY2YHySfjBQSGdPPy17exJ+bGNM+okr6YtIEDgfuNt9LsAZwKPuLiuBC93tpe5z3NcXu/svBR5S1W5VfQuoA05MxJfwuprGdmaWFzEh35/wYw+UWLZxfWOyQrw9/duArwJh9/kkYI+qRpZdqgdmuNszgG0A7uut7v4D7THeM0BElotItYhUNzc3j+CreFeooS2hN2VFsxLLxmSXYZO+iFwANKnqxiTEg6repaqLVHVRRUVFMj4yrXX19vPWzr0JvSkrmvX0jcku8YwXnAZ8TETOAwqAUuCHQJmI+N3efBDY7u6/HagC6kXED0wEdkW1R0S/xwxiy44OwgoLxqmnPyHfT6Ao1+bqG5Mlhu3pq+oNqhpU1Vk4F2KfU9VPA88DF7m7LQOecLdXuc9xX39Onakhq4BL3dk9s4G5wF8T9k08qsa9iDtePX2wEsvGZJOxXBn8N+AhEfkP4CXgHrf9HuABEakDWnBOFKjq6yLyCLAZ6AOuVdX+MXx+VqhtbKcg18dMdxhmPFSVFxJyp4UaY7xtRElfVV8AXnC3txJj9o2qdgEXD/L+m4GbRxpkNgs1tjGvsoQcX+LKLxwsGCji2ZomwmHFN46fY4xJPbsjN42pKjUN7eNyJ260qkAhPX1hdnZ0j+vnGGNSz5J+Gmvu6KZlb0/Ca+4cLFJi2aZtGuN9lvTTWChSQ3+8e/ruYio2bdMY77Okn8Yi5RfG68asiBllbk/fpm0a43mW9NNYqKGdqaUFBIrzxvVzCvNymDwh33r6xmQBS/pprKaxfVyKrMViJZaNyQ6W9NNUb3+YN5s6xv0ibkSVLaZiTFawpJ+m3tq5l57+MAvG+SJuRDBQyHt79tEftrr6xniZJf00VdMQKb+QpJ5+oIjefmVHW1dSPs8YkxqW9NNUqLGd3Bzh8MkTkvJ5AyWWbQaPMZ5mST9NhRraOKJiAnn+5PwVWYllY7KDJf00FWpsZ8E4VtY82PSyAkTsrlxjvM6Sfhpq7eylobVr3G/Kipbvz6GypMB6+sZ4nCX9NDSeC6EPpaq80Mb0jfE4S/ppKFLbPpnDO+AUXrOevjHeZkk/DYUa2wgU5TKlJD+pn1sVKKShdR+9/eGkfq4xJnks6aehSA19keQuaBIMFBFWaNhjc/WN8SpL+mkmHFZqG9uTdlNWtOBAiWUb1zfGqyzpp5l3WzrZ19uftPIL0apsMRVjPM+Sfqqpwu+/D3+4FYiqoZ+Cnv60iQXk+MQu5hrjYSNaGN0kWDgMa66D6hXO8/IjqGmYjwjMnZL8pO/P8TG1tMCmbRrjYcP29EWkQET+KiKviMjrIvJtt322iPxFROpE5GERyXPb893nde7rs6KOdYPbXisi54zXl8oI/X3wf190Ev5pX4Hpx8OqL9G0rY7Zk4opzMtJSVhV5YXW0zfGw+IZ3ukGzlDVY4BjgSUicjLw38CtqjoH2A1c4+5/DbDbbb/V3Q8RWQhcCrwPWALcLiKpyWyp1tcDj14Nmx6CM74BZ30HLroHwmEurb+JhVOLUhZaMFBkY/rGeNiwSV8dHe7TXPdHgTOAR932lcCF7vZS9znu64vFmXu4FHhIVbtV9S2gDjgxId8ik/Tug4cuh5pVsOQW+NC/OO3lh9N1znc5JryZK3p+lbLwqgJF7GjrpruvP2UxGGPGT1wXckUkR0ReBpqAdcCbwB5V7XN3qQdmuNszgG0A7uutwKTo9hjvif6s5SJSLSLVzc3NI/9G6ay7HR68GOqehY/+CE7+4gEv11Qs4bH+D3LSu3fD239MSYiREsvbbYjHGE+KK+mrar+qHgsEcXrn88crIFW9S1UXqeqiioqK8fqY5Nu3G+6/EN75E3zybvjAskN2CTW2883eq+ibeBg8/nnobEl6mFZi2RhvG9GUTVXdAzwPnAKUiUhk9k8Q2O5ubweqANzXJwK7ottjvMfbOprhvo9C4yb41ANw1EUxdws1tCH5JeRcfC90NMGqLztTOpOoyr1By8b1jfGmeGbvVIhImbtdCJwF1OAk/0j2WgY84W6vcp/jvv6cqqrbfqk7u2c2MBf4a6K+SNpqew/uOw921cFlD8H88wfdtaaxnSMrJ+ALHgdnfgtCT+6fzpkkU0oKyM2xufrGeFU88/SnASvdmTY+4BFVfVJENgMPich/AC8B97j73wM8ICJ1QAvOjB1U9XUReQTYDPQB16qqt68W7n4bVn7MGab5zONw2KmD7qrqlF84/+hpTsPJ/wBbn4e1X4OZp0DlwqSEnOMTZpRZiWVjvGrYpK+qm4DjYrRvJcbsG1XtAi4e5Fg3AzePPMwMtHOLk/B7O2HZEzDjA0Pu3tjWReu+XhZEauj7fHDhHXDHafDoZ2H585BbmITArcSyMV5mZRjGQ+NrcO+5EO6Fq1YPm/ABQg1ODf350TX0J0yBj98JzTWw9uvjFe0hnBu0rKdvjBdZ0k+0+o1w3/mQkwdXPwVT3x/X22oGWy1rzmI49R+h+h6o+U2io40pGChiZ0cP+3q8PfpmTDaypJ9Ib/8R7l8KhWVOwp88N+63hhramVFWSGlB7qEvnvENmH4cPPElaK1PYMCxRebqW2/fGO+xpJ8odc/Czz8JpdPh6qchcNiI3h5qbGPBYJU1/XnwyXsg3AePfR7C49sDD1qJZWM8y5J+ItQ8Cb+8DCbPgavXQOm0Eb29u6+fN5v3Dr0Q+qQj4PwfwLt/gt99b4wBD61qYDEVu5hrjNdY0h+rTb+CR66EacfAsiehePKID1HX1EF/WJk/3MIpx3wKjr4UfnsLvPPiKAMeXsWEfPL9Ppu2aYwHWdIfi433OeUSDjsVPvNrZyx/FGobnZk7gw7vRDv/exCYBY99btzKNIgIwYCVWDbGiyzpj9aLt8NvvgJzz4JP/wryR7/oSaixnTy/j1mTioffOb/EGd/vaITf/OO4lWmwEsvGeJMl/ZFShd99F9beAAs+Bp96cMw3TdU0tHFk5QT8OXH+dcw4Hhbf6Ezh3HjvmD57MLaYijHeZMsljoQqrP+2s57t0ZfC0p9Aztj/CL98xlw6e/qG3zHaKV9yyjQ8fYNTpmHKgjHHES0YKGJPZy/tXb2UxJpGaozJSNbTH4m/3e8k/EWfdUokJCDhA5w4u5yPzJsysjf5fHDhnc5wz6OfdRZnSaCqgJVYNsaLLOmPxKu/gooFztRJXxr80ZVUOom/aTM88+8JPfRAiWWbwWOMp6RB5soQnS3wzh+d0sgiqY5mv7lnOkM9G+527hdIkKD19I3xJEv68XpjLWgY5p+X6kgOtfhGmHYsPHFtwso0BIpyKc7LsRk8xniMJf141a6Gkukw7ZAq06nnz4OLVkB/Lzy+PCFlGpy5+lZi2RivsaQfj959ULce5p2bHmP5sUw6As7/vjME9fvvJ+SQVeW2mIoxXpOmGSzNbP2tsxhKOg7tRDvmUjjqEnjhv+DdP4/5cMFAEdt370OTvE6vMWb8WNKPR+1qyCuBWR9MdSRDE3F6+2UznTIN+3aP6XDBQCHt3X207utNUIDGmFSzpD+ccD/UPuWUW/Dnpzqa4RWUwidXQHsDrBpbmQabwWOM91jSH059NextdqZqZorgB2DxN6FmFfz1rlEfxubqG+M9lvSHU7safH6Yc2aqIxmZU74M886Dp/5t1Mss2mIqxnjPsElfRKpE5HkR2Swir4vIV9z2chFZJyJb3MeA2y4i8iMRqRORTSJyfNSxlrn7bxGRZeP3tRIotAZmnT7qsskp4/M51TiDi+DRa0ZVf39iYS6lBX4b3jHGQ+Lp6fcB16nqQuBk4FoRWQhcD6xX1bnAevc5wLnAXPdnOXAHOCcJ4EbgJOBE4MbIiSJtNb8Bu7bA/AtSHcno5BXB5Y84F3Z/+SloCo34EMFAkQ3vGOMhwyZ9VW1Q1b+52+1ADTADWAqsdHdbCVzobi8F7lfHn4EyEZkGnAOsU9UWVd0NrAOWJPTbJFrtaudx3rmpjWMsisrhisfAX+Cs4du6fURvtxLLxnjLiMb0RWQWcBzwF6BSVRvclxqBSnd7BrAt6m31bttg7ekrtMZZBnFiMNWRjE3gMPj0o9DVCg9eBPv2xP3WyF25NlffGG+IO+mLyATgMeCfVLUt+jV1MkJCsoKILBeRahGpbm5uTsQhR6ejCeo3wLwMmrUzlGlHw6UPws4t8NDl0NsV19uqAoXs6+1n196ecQ7QGJMMcSV9EcnFSfgPqurjbvMOd9gG97HJbd8OVEW9Pei2DdZ+AFW9S1UXqeqiioqKkXyXxKp9CtDMmqo5nMM/DB+/0ynV8Ov4avRUlbszeGxc3xhPiGf2jgD3ADWq+oOol1YBkRk4y4AnotqvdGfxnAy0usNAa4GzRSTgXsA9221LT6HVzgXQyvelOpLEOuoiOPtm2PyEs+rWMMM2doOWMd4Sz9JPpwGfAV4VkZfdtq8BtwCPiMg1wDvAJe5ra4DzgDqgE7gaQFVbROQmYIO733dUtSUh3yLRujtg6wvOClnpVDs/UU79knPH7os/htJpcPo/D7prMODeoGVz9Y3xhGGTvqr+ARgs8y2Osb8C1w5yrBXAipEEmBJvPgf93d4a2jnYWTdBxw549ltQMs0p1hZDcb6f8uI86+kb4xG2MHostWugMOAsOO5VPh8svd25YP3EtVA8edC7jqsCVmLZGK+wMgwH6++DN56GueckbOHztOXPg0/9HKYsgIevhPdeirlbpMSyMSbzWdI/2LsvOiWJ0712fqIUlDpz+IsnwYMXQ8vWQ3YJujdohcM2V9+YTGdJ/2C1ayAnH4445HKFd5VMhSsed6Zw/vyT0HHg/RHBQBE9/WGaO7pTFKAxJlEs6UdThdCTcPhHIH9CqqNJrslznTo9bQ3wi4udGUyuqoCVWDbGKyzpR9vxOux5N3uGdg5WdQJcfB80bIJfLXMWWsfm6hvjJZb0o9WuAQSOzOACa2M1bwlccCvUPTuw8lbQevrGeIbHp6eMUGg1BE+Aksrh9/WyDyyD9kZ44T+hdBoFi79JRUm+3aBljAdY0o9orYeGl+HMb6U6kvTw4a9C+3vw++9DyTSqAgtteMcYD7DhnYjap5xHr1TVHCsROO/7zpKLa/6V83OrradvjAdY0o8IrYZJc6DiyFRHkj5y/O6Siyew7L3/YEbry/T1h1MdlTFmDCzpg7OoyNu/93atndHKK4LLH2Zv0Qx+6v8uO996JdURGWPGwJI+ODNVwn02tDOYonLeOHMlXeQRePyyES+5aIxJH5b0wRnaKa6A4KJUR5K2psycw7Ke6/H1tDt37Y5gyUVjTPqwpN/X4/T0j1wCvpxUR5O2pk0spJaZ/N+878KuLfD45+NaecsYk14s6b/9e+hug/kXpDqStJbn9zGttIAXdSGc+z+w5Rl4/uZUh2WMGSFL+qHVkFvkrB9rhhQMFDlz9U+4Bj5wlTOH//VfpzosY8wIZHfSV3Xm5x9xBuQWpjqatBcsL6Q+Uorh3P+BqpPg//4BGl9LbWDGmLhld9J/7yXnrlObqhmXYKCIxrYuevrC4M+HS+6Hgonw0OXQmZ7LHRtjDpTdST+0GsTnXMQ1w6oKFBJWaGh1yzGUTHVW3mpvgEevdlYdM8aktexO+rVrYOapUFSe6kgyQswSy8FFTlXOrS/AszemJjBjTNyyN+m3vAVNm7O3dv4oVJUPUmL5uCvgxL+HF38Mmx5JQWTGmHgNm/RFZIWINInIa1Ft5SKyTkS2uI8Bt11E5EciUicim0Tk+Kj3LHP33yIiy8bn64xA7RrncZ4l/XhNLS0gxyexq22eczPM+iCs+vKgC6wbY1Ivnp7+fcDBg97XA+tVdS6w3n0OcC4w1/1ZDtwBzkkCuBE4CTgRuDFyokiZ0GqY8j4on53SMDKJP8fH9LKC2NU2c3KdVbeKK+ChKw5ZZ9cYkx6GTfqq+jvg4KkZS4GV7vZK4MKo9vvV8WegTESmAecA61S1RVV3A+s49ESSPHt3wbsv2tDOKFQFigZfQat4Mlz6IHTuOmC5RWNM+hjtmH6lqja4241AZKmpGcC2qP3q3bbB2g8hIstFpFpEqpubx6m3uGUtaNiGdkYhGCgcejGVacfAx/4X3vkjPH1D8gIzxsRlzBdyVVUBTUAskePdpaqLVHVRRUVFog57oNBqKJkO048bn+N7WFWgiKb2brp6h6i7c/TFcOqXYcPP4G/3Jy84Y8ywRpv0d7jDNriPTW77dqAqar+g2zZYe/L17oM3n3OGdkRSEkImC7ozeLbvGWbpxDO/7dzpvPo62LYhCZEZY+Ix2qS/CojMwFkGPBHVfqU7i+dkoNUdBloLnC0iAfcC7tluW/JtfQF6O21oZ5TmTy0F4MfP1eH8kjcIX46z6lbpdHj4CmhrGHxfY0zSxDNl85fAi8A8EakXkWuAW4CzRGQLcKb7HGANsBWoA34G/AOAqrYANwEb3J/vuG3JF1oN+aXO9EIzYgumlXLdWUfy65e288P1W4beuagcLv0FdLfDI5+Bvu7kBGmMGZR/uB1U9bJBXlocY18Frh3kOCuAFSOKLtHC/fDG0zD3LPDnpTSUTPalM+bw9q5Obnt2C4dNKuLjxwUH37nyffDxO+CRK52hno/9rw2rGZNC2XVHbv0G2NtsQztjJCL81yeO4uTDy/nqo5v4y9ZdQ79h4VL40L/CSw/AhruTE6QxJqbsSvqh1eDLdXr6Zkzy/D5+esUiZpYXsfyBjbzZ3DH0Gz7yNaew3dPXw9t/TE6QxphDZE/SV3WS/qzTnXLAZswmFuVy71Un4vcJn71vA7s6hhiz9/ngE3dBYLYz1NNan7xAjTEDsifp73wDWt602vkJNnNSET9btojG1i6WP7Bx6Pn7BROdC7v9PU4N/t5hpn0aYxIue5J+aLXzaOP5CXf8zAA/uORYNr6zm3/51SuEw0NM5aw4Ej7xM2jYBL/5ivMbmDEmabIn6deugWnHwsSY1R/MGJ1/9DT+bcl8ntzUwA/WvTH0zvOWwN99HTY9DH++PTkBGmOAbEn67Y1QX21DO+PsCx8+nEtPqOLHz9fxSPW2oXf+4HWw4KPwzL/Dm88nJ0BjTJYk/dqnALWkP85EhJsufD+nz5nM1x5/lT/V7Rx8Z58PLrwTKuY7Sy22vJW8QI3JYlmS9NdA2WEwZWGqI/G83Bwft19xPLMnF/P3P99IXVP74OsgWRAAAAp5SURBVDvnT3BKMWsYHvo09OxNXqDGZCnvJ/3uDtj6W6eXb3eCJkVpQS73Xn0C+f4crrp3A83tQ0zlLD8cLloBzTXwq6ugqSZpcRqTjbyf9N9cD/3dNmsnyYKBIu5ZtoidHd18/v7qoadyzjkTzvkvqFsPt58Md30E/nIXdKamPJMxXub9pB9aDYUBmHlKqiPJOsdUlXHbp47jlfo9/L9HXh56KufJX4DrQnDOf0J/Hzz1r/C9I50KnaE1tgqXMQni7aTf3wtvrHVu/88ZtracGQdL3j+Vr5+3gDWvNvLfa0ND7zxhCpxyLXzxD/CFP8CJy+GdF+Ghy+D7852VuBpfTU7gxniUtzPhuy9C1x4b2kmxa06fzdu79vLT327lsPJiLj9p5vBvmnoULDkKzvo21D0LL/8C/vozZ15/5VFw7GVw1CUwYZxWVzPGo7yd9ENrICffWcHJpIyI8K2Pvo9tLfv4xhOvEQwU8qEj40zWObkw71znp7MFXnvMOQGs/Ro88w2Ye7ZzAjhyCfjzx/eLGOMBMuTqRym2aNEira6uHt2bVeG2o6FyIVz+cGIDM6PS3tXLxXe+SP3ufTz2xVOZN7Vk9AdrCsErv4BXHoaORue6zfsvck4A04+3mVomq4nIRlVdFOs1747p73gNWt+1oZ00UlKQy4qrTqA4P4fP3reBprau0R9synw46zvwz6/Dpx9zfpt76QH42Rnwk5PgD7fZEo3GxODdpB9aA4gzLGDSxvSyQu5ZdgK7O3v43P3VdPb0je2AOX6Ye6Yz1/+6WrjgNigsg2dvhFsXwgOfcK4FvPMnmwJqDF4e3rnzg5BbCNc8k9igTEKsr9nB5++vZvGCSu684gPk+BI8HLOzDl75pVPUrTWqDtCESpiyACoWOI9TFjilIApKE/v5xqTQUMM73kz6e7bBbe+HM78Np/9T4gMzCXHfH9/iW7/ZzOdOn82/XzBOJTJUnQVbmkPO3b5NNc7dv8210Nu5f7/SoDNkFH1CqJgHecXjE5cx42iopO/N2Ts9e53ZHFZgLa1dddps3t7Vyd1/eIvDJhXxmVNmJf5DRKCsyvmJXiYzHIY977gng83OheHmGnjr984d3M6boWymU7NpynznsWI+TD4ScgsSH6sxSZD0nr6ILAF+COQAd6vqLYPtO6bhHZMR+sPK3z9QzXOhJn5y+fGcOmcyJfl+fIke7ok7oD7Y/bZzAmiK+tm1BcLu9QfxOcs+TpgCeROc3wbyJ0BeifsYaStxtgfaDtq2GwbNOEmb4R0RyQHeAM4C6oENwGWqujnW/pb0s8Pe7j4u+emLvP5e20BbSb6fkgI/JQW5lBY6jyUFfkrdx4HnhZH2yGvO86K8HCSR0zb7e2HXm85vBc0h56ezBXo6nKJ+PR3Ob5jd7UCc/6f8BVEnghLnRJFX7NxvkJMLOXngy92/nZM7SHuecwKJbPuitiPv8eWCL8c5Yfn87naO8xi9PdDmd/eNtPn3v9+mw6a9dBreORGoU9WtACLyELAUiJn0TXYozvfzi8+dzNrNjbTt66Wtq4/2rl7a9jmP7V197Gjroq5p//O+oer4ADk+cU8OfvL9Oajq/lSs+9NypD3S91F0/7bu38d9G6oTgZNQTuSLHz6Cq06bfeAHqzrXCgZOBFEnhe5258Qw0NbuPu7d//q+3c4JJtzrrCXcH/0Y2e4h7hPLeBDf/hPBwElAoh456PkQj+Ib4r3j/kXcBzm0bbD2uPZNkDlnwjk3J/ywyU76M4DoJZXqgZOidxCR5cBygJkz47hd33jCxKJcLllUFde+qsq+3n7a3ZNDa9TJoc19HHi+r5fefjdByv7/piKCsP//qrhtke2B/+OIk5sG3ue0ARw2KcZFXpH9PXYqR/aHMBLh/tgng3Df/u3+qO1wr3MdQ/udfcL97nastv4Dtwdt63NOcqo4Z9IxPB7cNt4GRjg0Rttg7XHsm0il08flsGk3qKiqdwF3gTO8k+JwTBoSEYry/BTl+akszdILqr4c8BU605KNGYFk35y1HYjuzgXdNmOMMUmQ7KS/AZgrIrNFJA+4FFiV5BiMMSZrJXV4R1X7RORLwFqcKZsrVPX1ZMZgjDHZLOlj+qq6BliT7M81xhjj5YJrxhhjDmFJ3xhjsoglfWOMySKW9I0xJoukdWllEWkG3hnDISYDOxMUznjLpFghs+K1WMdPJsWbSbHC2OI9TFVjLkSd1kl/rESkerCiQ+kmk2KFzIrXYh0/mRRvJsUK4xevDe8YY0wWsaRvjDFZxOtJ/65UBzACmRQrZFa8Fuv4yaR4MylWGKd4PT2mb4wx5kBe7+kbY4yJYknfGGOyiCeTvogsEZFaEakTketTHc9QRKRKRJ4Xkc0i8rqIfCXVMQ1HRHJE5CUReTLVsQxHRMpE5FERCYlIjYickuqYBiMi/+z+G3hNRH4pImm1QoyIrBCRJhF5LaqtXETWicgW9zGQyhgjBon1u+6/g00i8msRKUtljNFixRv12nUioiIyORGf5bmk7y6+/hPgXGAhcJmILExtVEPqA65T1YXAycC1aR4vwFeAmlQHEacfAk+r6nzgGNI0bhGZAfwjsEhV349TevzS1EZ1iPuAJQe1XQ+sV9W5wHr3eTq4j0NjXQe8X1WPBt4Abkh2UEO4j0PjRUSqgLOBdxP1QZ5L+kQtvq6qPUBk8fW0pKoNqvo3d7sdJynNSG1UgxORIHA+cHeqYxmOiEwEPgTcA6CqPaq6J7VRDckPFIqIHygC3ktxPAdQ1d8BLQc1LwVWutsrgQuTGtQgYsWqqs+oap/79M84K/elhUH+bAFuBb5KAhfi9WLSj7X4etom0WgiMgs4DvhLaiMZ0m04/wjDqQ4kDrOBZuBedzjqbhGJsZp56qnqduB7OD26BqBVVZ9JbVRxqVTVBne7kXFdDT6hPgs8leoghiIiS4HtqvpKIo/rxaSfkURkAvAY8E+q2pbqeGIRkQuAJlXdmOpY4uQHjgfuUNXjgL2kz/DDAdyx8KU4J6rpQLGIXJHaqEZGnfnfaT8HXES+jjOs+mCqYxmMiBQBXwO+mehjezHpZ9zi6yKSi5PwH1TVx1MdzxBOAz4mIm/jDJudISI/T21IQ6oH6lU18pvTozgngXR0JvCWqjarai/wOHBqimOKxw4RmQbgPjalOJ4hichVwAXApzW9b1I6AqcD8Ir7/y0I/E1Epo71wF5M+hm1+LqICM6Yc42q/iDV8QxFVW9Q1aCqzsL5c31OVdO2N6qqjcA2EZnnNi0GNqcwpKG8C5wsIkXuv4nFpOlF54OsApa528uAJ1IYy5BEZAnO0OTHVLUz1fEMRVVfVdUpqjrL/f9WDxzv/pseE88lffdCTWTx9RrgkTRffP004DM4veaX3Z/zUh2Uh3wZeFBENgHHAv+Z4nhicn8beRT4G/Aqzv/NtCobICK/BF4E5olIvYhcA9wCnCUiW3B+W7kllTFGDBLrj4ESYJ37/+zOlAYZZZB4x+ez0vs3HGOMMYnkuZ6+McaYwVnSN8aYLGJJ3xhjsoglfWOMySKW9I0xJotY0jfGmCxiSd8YY7LI/wfpaZNjMImHLQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ghRgQvOnVewZ", + "outputId": "b9c72b93-0728-4a11-8ecc-b38a30062f27" + }, + "source": [ + "trn_url_parts_df[trn_url_parts_df.netloc_level_cnt==7].netloc.tolist()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['bmcinfectdis.biomedcentral.com.preview-live.oscarjournals.springer.com',\n", + " '0-www.loc.gov.oasys.lib.oxy.edu',\n", + " 'www.bennettandbelfort.com.php73-36.phx1-1.websitetestlink.com']" + ] + }, + "metadata": {}, + "execution_count": null + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2cmQUCorT4sS", + "outputId": "5813272b-0007-43b2-c817-c57302a8ab76" + }, + "source": [ + "trn_url_parts_df[trn_url_parts_df.path_level_cnt==14].path.tolist()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['/MetroKids/Attractions-and-Events/Attractions-Guide/index.php/alpha/P/category/Theaters/ages/Parents/city/Malvern/county/NJ+-+Burlington+County/']" + ] + }, + "metadata": {}, + "execution_count": null + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "sR1h230cWTdo", + "outputId": "ce8d4cb8-2730-4a34-f4d0-2b5cd84a944b" + }, + "source": [ + "trn_url_parts_df = trn_url_parts_df.assign(\n", + " yearlike_path=trn_url_parts_df.path.str.contains(r\"20[0-2][0-9]|19\\d\\d\")\n", + ")\n", + "print(\n", + " f\"{len(trn_url_parts_df[trn_url_parts_df.yearlike_path].index)/len(trn_url_parts_df.index):.2%}\"\n", + ")\n", + "trn_url_parts_df.yearlike_path.value_counts().to_dict()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "22.52%\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "{False: 7748, True: 2252}" + ] + }, + "metadata": {}, + "execution_count": null + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Tf3hlUV1WTdt", + "outputId": "544100f9-0122-42b5-e309-e0a5f8daf00f" + }, + "source": [ + "trn_url_parts_df[trn_url_parts_df.yearlike_path].path[:10].tolist()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['/news/2005/03/22/on-time-and-on-budget/',\n", + " '/2013/01/19/kitchen-tip-7-how-to-cook-pasta/',\n", + " '/2014/08/18/us-pay-tv-providers-shed-300000-subs-in-q2/',\n", + " '/2011/11/24/reflections-thanksgiving-2011/',\n", + " '/2019/',\n", + " '/oem-parts/2005-suzuki-dr200se-gasket-set/o/m16072sch590528',\n", + " '/Products/Genuine-Joe-Linen-like-Table-Skirts__GJO11915.aspx',\n", + " '/listing/485271256/vintage-90s-1995-58th-annual-postal',\n", + " '/2013/01/08/calendar-cat/',\n", + " '/2012/06/new-model-army-equipment-stolen-can-you-help/']" + ] + }, + "metadata": {}, + "execution_count": null + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "OG8YdBMqWTdu", + "outputId": "8e1a083e-df6f-4519-e47f-a08f250b4177" + }, + "source": [ + "import warnings\n", + "\n", + "\n", + "warnings.simplefilter(action=\"ignore\") # Just don't want to see `UserWarning` about regex group\n", + "trn_url_parts_df = trn_url_parts_df.assign(\n", + " datelike_path=trn_url_parts_df.path.str.contains(\n", + " r\"(20[0-2][0-9]|19\\d\\d)[/\\-](0?[1-9]|11|12)[/\\-](0?[1-9]|[12]\\d|3[01])\"\n", + " )\n", + ")\n", + "print(\n", + " f\"{len(trn_url_parts_df[trn_url_parts_df.datelike_path].index)/len(trn_url_parts_df.index):.2%}\"\n", + ")\n", + "trn_url_parts_df.datelike_path.value_counts().to_dict()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "7.34%\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "{False: 9266, True: 734}" + ] + }, + "metadata": {}, + "execution_count": null + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vN1sK3uzWTdu", + "outputId": "1cf1dbef-5e98-4112-e20c-05f1a13b3f80" + }, + "source": [ + "trn_url_parts_df[trn_url_parts_df.datelike_path].path[:10].tolist()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['/news/2005/03/22/on-time-and-on-budget/',\n", + " '/2013/01/19/kitchen-tip-7-how-to-cook-pasta/',\n", + " '/2014/08/18/us-pay-tv-providers-shed-300000-subs-in-q2/',\n", + " '/2011/11/24/reflections-thanksgiving-2011/',\n", + " '/2013/01/08/calendar-cat/',\n", + " '/2016/02/04/the-jesus-revolution/',\n", + " '/2016/03/18/colorado-avalanche-vs-calgary-flames-nhl-betting-hockey-odds-pick-and-prediction/',\n", + " '/news/2013/08/06/communities-should-be-paid-for-wind-and-solar-in-their-neighbourhood/',\n", + " '/2017/11/20/maurizio-cannavacciuolo/',\n", + " '/index.php/2014/02/19/showing-a-little-love/']" + ] + }, + "metadata": {}, + "execution_count": null + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wexEuNAuOTrI" + }, + "source": [ + "## Deprecated Basic EDA of C4 samples.tgz\n", + "Old code for raw data of C4 samples.tgz (uncompressed as `dst/en_meta.json`, for example)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cEPExQdIbqN1" + }, + "source": [ + "### English" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "m9e6X_9cSFPr" + }, + "source": [ + "import json\n", + "import pandas as pd\n", + "\n", + "from collections import namedtuple\n", + "from glob import glob\n", + "from urllib.parse import unquote_plus, urlsplit" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Otf_HTdUSX8y" + }, + "source": [ + "%cd /content/drive/MyDrive/colab_data/dst" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "ucRZz8zRxVFW" + }, + "source": [ + "with open('en_meta.json') as en_meta_f:\n", + " en_meta_json = json.load(\n", + " en_meta_f,\n", + " object_hook=lambda d: namedtuple(\n", + " 'Meta',\n", + " map(lambda k: k.replace('-', '_'), d.keys())\n", + " )(*d.values())\n", + " )" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "8JyoUMQKxttO" + }, + "source": [ + "en_meta_offset_uri_map = {\n", + " row.offset: row.headers.warc_target_uri for row in en_meta_json\n", + "}" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "5ml3w27g6TBb" + }, + "source": [ + "en_uri_parts_map = {\n", + " uri: urlsplit(uri) for uri in en_meta_offset_uri_map.values()\n", + "}" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "d-_TysttCbFD" + }, + "source": [ + "i = 0\n", + "for (uri, parts) in en_uri_parts_map.items():\n", + " if '%' in parts.path:\n", + " print(f\"{i:02d} {uri} -> {unquote_plus(parts.path)} {unquote_plus(parts.query)} {unquote_plus(parts.fragment)}\")\n", + " i += 1\n", + " if i > 50:\n", + " break" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "V1cbOBx2w2bd" + }, + "source": [ + "en_uri_parts_df = pd.DataFrame(\n", + " data=[(\n", + " uri,\n", + " unquote_plus(parts.path),\n", + " sum(1 for _ in filter(None, parts.path.split('/'))),\n", + " unquote_plus(parts.query),\n", + " unquote_plus(parts.fragment)\n", + " ) for uri, parts in en_uri_parts_map.items()],\n", + " columns=['uri', 'path', 'path_level_cnt', 'query', 'fragment']\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6lisgI0hbLUd" + }, + "source": [ + "#### Number of Path Levels" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "OnRoyOSQxEpj" + }, + "source": [ + "en_uri_parts_df.describe(percentiles=[.25, .5, .75, .95, .975])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "dSbh_K4FKIjJ" + }, + "source": [ + "en_uri_parts_df.path_level_cnt.value_counts().sort_index().plot()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "uvslgW7mFn71" + }, + "source": [ + "en_uri_parts_df[en_uri_parts_df.path_level_cnt==33].uri.values" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "w9LU95g1IHvz" + }, + "source": [ + "en_uri_parts_df[en_uri_parts_df.path_level_cnt==5].path.values" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QluA_8eLbTqr" + }, + "source": [ + "#### Year and Date" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Wjs8GczPOSPp" + }, + "source": [ + "en_uri_parts_df = en_uri_parts_df.assign(\n", + " yearlike_path=en_uri_parts_df.path.str.contains(r\"20[0-2][0-9]|19\\d\\d\")\n", + ")\n", + "display(f\"{len(en_uri_parts_df[en_uri_parts_df.yearlike_path].index)/len(en_uri_parts_df.index):.2%}\")\n", + "en_uri_parts_df.yearlike_path.value_counts()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "vx5ls4MESADX" + }, + "source": [ + "en_uri_parts_df[en_uri_parts_df.yearlike_path].path.values" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Zck_PuJ0PoCx" + }, + "source": [ + "en_uri_parts_df = en_uri_parts_df.assign(\n", + " en_uri_parts_df=en_uri_parts_df.path.str.contains(\n", + " r\"(20[0-2][0-9]|19\\d\\d)[/\\-](0?[1-9]|11|12)[/\\-](0?[1-9]|[12]\\d|3[01])\"\n", + " )\n", + ")\n", + "display(f\"{len(en_uri_parts_df[en_uri_parts_df.datelike_path].index)/len(en_uri_parts_df.index):.2%}\")\n", + "en_uri_parts_df.datelike_path.value_counts()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "BrJFJtjEPuMx" + }, + "source": [ + "en_uri_parts_df[en_uri_parts_df.datelike_path].path.values" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "__lnPt_ya_SS" + }, + "source": [ + "### All" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "SnoCgIxkTVWs" + }, + "source": [ + "meta_jsons = []\n", + "for meta_json_fname in glob('??_meta.json'):\n", + " with open(meta_json_fname) as meta_json_f:\n", + " meta_jsons.extend(\n", + " json.load(\n", + " meta_json_f,\n", + " object_hook=lambda d: namedtuple(\n", + " 'Meta',\n", + " map(lambda k: k.replace('-', '_'), d.keys())\n", + " )(*d.values())\n", + " )\n", + " )" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "2dz9enqbT6iK" + }, + "source": [ + "meta_offset_uri_map = {\n", + " row.offset: row.headers.warc_target_uri for row in meta_jsons\n", + "}" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "1cqTjbtjV1oR" + }, + "source": [ + "uri_parts_map = {\n", + " uri: urlsplit(uri) for uri in meta_offset_uri_map.values()\n", + "}" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "qgLPRtDnVVEk" + }, + "source": [ + "uri_parts_df = pd.DataFrame(\n", + " data=[(\n", + " uri,\n", + " unquote_plus(parts.path),\n", + " sum(1 for _ in filter(None, parts.path.split('/'))),\n", + " unquote_plus(parts.query),\n", + " unquote_plus(parts.fragment)\n", + " ) for uri, parts in uri_parts_map.items()],\n", + " columns=['uri', 'path', 'path_level_cnt', 'query', 'fragment']\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0nXMJuiMbcBx" + }, + "source": [ + "#### Number of Path Levels" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "3y7CVd1iV_yN" + }, + "source": [ + "uri_parts_df.describe(percentiles=[.25, .5, .75, .95, .975])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "88eQb3RFWGnk" + }, + "source": [ + "uri_parts_df.path_level_cnt.value_counts().sort_index().plot()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "isBR1jmUWMb0" + }, + "source": [ + "uri_parts_df[uri_parts_df.path_level_cnt==61].uri.values" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "j3EMRvoAbf9Y" + }, + "source": [ + "#### Year and Date" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "fLdePo9-Wqx8" + }, + "source": [ + "uri_parts_df = uri_parts_df.assign(\n", + " yearlike_path=df_uri_parts.path.str.contains(r\"20[0-2][0-9]|19\\d\\d\")\n", + ")\n", + "display(f\"{len(uri_parts_df[uri_parts_df.yearlike_path].index)/len(uri_parts_df.index):.2%}\")\n", + "uri_parts_df.yearlike_path.value_counts()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "iffYbZ_BXNjE" + }, + "source": [ + "uri_parts_df = uri_parts_df.assign(\n", + " datelike_path=uri_parts_df.path.str.contains(\n", + " r\"(20[0-2][0-9]|19\\d\\d)[/\\-](0?[1-9]|11|12)[/\\-](0?[1-9]|[12]\\d|3[01])\"\n", + " )\n", + ")\n", + "display(f\"{len(uri_parts_df[uri_parts_df.datelike_path].index)/len(uri_parts_df.index):.2%}\")\n", + "uri_parts_df.datelike_path.value_counts()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "rWJKdnZFZSvH" + }, + "source": [ + "" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file