[Fix] Update setup.py and MANIFEST.in (InternLM#45)

* update MANIFEST.in

* update setup.py

* add pip install xtuner[deepspeed]

MANIFEST.in

@@ -1,5 +1,2 @@
-include requirements/*.txt
-include xtuner/.mim/model-index.yml
-include xtuner/.mim/demo/*/*
-recursive-include xtuner/.mim/configs *.py *.yml
-recursive-include xtuner/.mim/tools *.sh *.py
+recursive-include xtuner/configs *.py *.yml *.json
+recursive-include xtuner/tools *.sh *.py

requirements.txt

@@ -1,12 +1,2 @@
-bitsandbytes>=0.40.0
-datasets
-einops
-lagent
-mmengine>=0.8.4
-peft>=0.4.0
-scipy
-SentencePiece
-tiktoken
-torch
-transformers>=4.31.0
-transformers_stream_generator
+-r requirements/runtime.txt
+-r requirements/deepspeed.txt

requirements/deepspeed.txt

@@ -0,0 +1,2 @@
+deepspeed
+mpi4py-mpich

requirements/runtime.txt

@@ -0,0 +1,12 @@
+bitsandbytes>=0.40.0
+datasets
+einops
+lagent
+mmengine>=0.8.4
+peft>=0.4.0
+scipy
+SentencePiece
+tiktoken
+torch
+transformers>=4.31.0
+transformers_stream_generator

setup.py

@@ -1,11 +1,5 @@
 #!/usr/bin/env python
 # Copyright (c) OpenMMLab. All rights reserved.
-import os
-import os.path as osp
-import platform
-import shutil
-import sys
-import warnings
 from setuptools import find_packages, setup
 
 
@@ -101,85 +95,38 @@ def gen_packages_items():
     return packages
 
 
-def add_mim_extension():
-    """Add extra files that are required to support MIM into the package.
-
-    These files will be added by creating a symlink to the originals if the
-    package is installed in `editable` mode (e.g. pip install -e .), or by
-    copying from the originals otherwise.
-    """
-
-    # parse installment mode
-    if 'develop' in sys.argv:
-        # installed by `pip install -e .`
-        if platform.system() == 'Windows':
-            # set `copy` mode here since symlink fails on Windows.
-            mode = 'copy'
-        else:
-            mode = 'symlink'
-    elif 'sdist' in sys.argv or 'bdist_wheel' in sys.argv:
-        # installed by `pip install .`
-        # or create source distribution by `python setup.py sdist`
-        mode = 'copy'
-    else:
-        return
-
-    filenames = ['tools', 'configs', 'demo', 'model-index.yml']
-    repo_path = osp.dirname(__file__)
-    mim_path = osp.join(repo_path, 'xtuner', '.mim')
-    os.makedirs(mim_path, exist_ok=True)
-
-    for filename in filenames:
-        if osp.exists(filename):
-            src_path = osp.join(repo_path, filename)
-            tar_path = osp.join(mim_path, filename)
-
-            if osp.isfile(tar_path) or osp.islink(tar_path):
-                os.remove(tar_path)
-            elif osp.isdir(tar_path):
-                shutil.rmtree(tar_path)
-
-            if mode == 'symlink':
-                src_relpath = osp.relpath(src_path, osp.dirname(tar_path))
-                os.symlink(src_relpath, tar_path)
-            elif mode == 'copy':
-                if osp.isfile(src_path):
-                    shutil.copyfile(src_path, tar_path)
-                elif osp.isdir(src_path):
-                    shutil.copytree(src_path, tar_path)
-                else:
-                    warnings.warn(f'Cannot copy file {src_path}.')
-            else:
-                raise ValueError(f'Invalid mode {mode}')
-
-
 if __name__ == '__main__':
-    add_mim_extension()
     setup(
         name='xtuner',
         version=get_version(),
-        description='xTuner',
+        description='A toolkit for efficiently fine-tuning LLM',
         long_description=readme(),
         long_description_content_type='text/markdown',
         author='{author name}',
         author_email='{your email}',
-        keywords='xtuner, template',
-        url='https://github.com/user_name/xtuner',
-        packages=find_packages(exclude=('configs', 'tools', 'demo')),
+        keywords='large language model, parameter-efficient fine-tuning',
+        url='https://github.com/InternLM/xtuner',
+        packages=find_packages(),
         include_package_data=True,
         classifiers=[
             'Development Status :: 4 - Beta',
             'License :: OSI Approved :: Apache Software License',
             'Operating System :: OS Independent',
             'Programming Language :: Python :: 3',
-            'Programming Language :: Python :: 3.7',
             'Programming Language :: Python :: 3.8',
             'Programming Language :: Python :: 3.9',
             'Programming Language :: Python :: 3.10',
             'Topic :: Utilities',
         ],
-        python_requires='>=3.7',
+        python_requires='>=3.8',
         license='Apache License 2.0',
-        install_requires=parse_requirements('requirements.txt'),
+        install_requires=parse_requirements('requirements/runtime.txt'),
+        extras_require={
+            'all':
+            parse_requirements('requirements.txt'),
+            'deepspeed':
+            parse_requirements('requirements/runtime.txt') +
+            parse_requirements('requirements/deepspeed.txt')
+        },
         zip_safe=False,
         entry_points={'console_scripts': ['xtuner = xtuner:cli']})