Merge pull request #4 from cfhamlet/develop

v0.0.2

Author: cfhamlet

README.md

@@ -23,10 +23,22 @@ pip install os-scrapy
 
 ### Command line
 
+The command is same as scrapy
+
 ```
-os-scrapy
+os-scrapy -h
 ```
 
+### CRAWLER_CLASS
+
+os-scrapy support config ``CRAWLER_CLASS`` replace the default ``scrapy.crawler.Crawler``. It can be configured in ``settings.py`` file or ``os-scrapy crawl -c <your_crawler_class> <spider_name>``
+
+
+## Enhanced ``startproject``
+
+Thanks to [os-scrapy-cookiecutter](https://github.com/cfhamlet/os-scrapy-cookiecutter) a ``-p`` option can be used ``os-scrapy startproject -p <project_name>``, which indicate create project as well as python package. 
+
+
 ## Unit Tests
 
 ```
@@ -35,4 +47,4 @@ tox
 
 ## License
 
-MIT licensed.
+MIT licensed.

requirements/requirements.txt

@@ -1 +1 @@
-Scrapy==2.0.1
+os-scrapy-cookiecutter>=0.0.2

script/lint.sh scripts/lint.shscript/lint.sh renamed to scripts/lint.sh

@@ -10,6 +10,8 @@ set -x
 
 pip install -r requirements/requirements-lint.txt
 
-${PREFIX}autoflake --in-place --recursive --remove-all-unused-imports --remove-unused-variables src tests
-${PREFIX}black --exclude=".pyi$" src tests
-${PREFIX}isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --combine-as --line-width 88 --recursive --apply src tests
+FILES="src tests setup.py"
+
+${PREFIX}autoflake --in-place --recursive --remove-all-unused-imports --remove-unused-variables ${FILES}
+${PREFIX}black --exclude=".pyi$" ${FILES}
+${PREFIX}isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --combine-as --line-width 88 --recursive --apply ${FILES}

setup.py

@@ -1,11 +1,12 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 
 def read(*filenames, **kwargs):
     import io
     from os.path import join, dirname
-    encoding = kwargs.get('encoding', 'utf-8')
-    sep = kwargs.get('sep', '\n')
+
+    encoding = kwargs.get("encoding", "utf-8")
+    sep = kwargs.get("sep", "\n")
     buf = []
     for filename in filenames:
         with io.open(join(dirname(__file__), filename), encoding=encoding) as f:
@@ -14,36 +15,33 @@ def read(*filenames, **kwargs):
 
 
 setup(
-    name='os-scrapy',
-    version=read('src/os_scrapy/VERSION').strip(),
-    packages=find_packages(where='src'),
-    package_dir={'': 'src'},
+    name="os-scrapy",
+    version=read("src/os_scrapy/VERSION").strip(),
+    packages=find_packages(where="src"),
+    package_dir={"": "src"},
     include_package_data=True,
-    license='MIT License',
-    description='Ozzy \'s Scrapy',
-    long_description=open('README.md').read(),
-    long_description_content_type='text/markdown',
-    author='Ozzy',
-    author_email='cfhamlet@gmail.com',
-    url='https://github.com/cfhamlet/os-scrapy',
-    install_requires=open('requirements/requirements.txt').read().split('\n'),
-    python_requires='>=3.6',
+    license="MIT License",
+    description="Ozzy 's Scrapy",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    author="Ozzy",
+    author_email="cfhamlet@gmail.com",
+    url="https://github.com/cfhamlet/os-scrapy",
+    install_requires=open("requirements/requirements.txt").read().split("\n"),
+    python_requires=">=3.6",
     zip_safe=False,
-    entry_points={
-        'console_scripts': ['os-scrapy = os_scrapy.cmdline:execute']
-    },
-    extras_require={
-        'ujson': ['ujson'],
-    },
+    entry_points={"console_scripts": ["os-scrapy = os_scrapy.cmdline:execute"]},
+    extras_require={"ujson": ["ujson"],},
     classifiers=[
-        'Development Status :: 2 - Pre-Alpha',
-        'Intended Audience :: Developers',
-        'License :: OSI Approved :: MIT License',
-        'Natural Language :: English',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3 :: Only',
-        'Programming Language :: Python :: Implementation :: CPython',
-        'Programming Language :: Python :: Implementation :: PyPy',
-    ])
+        "Development Status :: 2 - Pre-Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: MIT License",
+        "Natural Language :: English",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: Implementation :: CPython",
+        "Programming Language :: Python :: Implementation :: PyPy",
+    ],
+)

src/os_scrapy/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

src/os_scrapy/commands/startproject.py

@@ -0,0 +1,54 @@
+from os.path import abspath, exists, join
+
+import os_scrapy_cookiecutter
+from cookiecutter.main import cookiecutter
+from scrapy.commands.startproject import Command as ScrapyCommand
+from scrapy.exceptions import UsageError
+
+
+class Command(ScrapyCommand):
+    def add_options(self, parser):
+        super(Command, self).add_options(parser)
+        parser.add_option(
+            "-p",
+            "--package",
+            dest="package",
+            action="store_true",
+            help="create project as package",
+        )
+
+    def run(self, args, opts):
+        if not opts.package:
+            return super(Command, self).run(args, opts)
+
+        if len(args) not in (1, 2):
+            raise UsageError()
+
+        project_name = args[0]
+        project_dir = args[0]
+
+        if len(args) == 2:
+            project_dir = args[1]
+
+        if exists(join(project_dir, "scrapy.cfg")):
+            self.exitcode = 1
+            print("Error: scrapy.cfg already exists in %s" % abspath(project_dir))
+            return
+
+        if not self._is_valid_name(project_name):
+            self.exitcode = 1
+            return
+
+        try:
+            cookiecutter(
+                os_scrapy_cookiecutter.TEMPLATE_DIR,
+                no_input=True,
+                extra_context={
+                    "project_name": project_name,
+                    "project_dir": project_dir,
+                },
+            )
+        except Exception as e:
+            self.exitcode = 1
+            print(f"Error: create project with cookiecutter {e}")
+            return