[mod] remove buildout/makefile infrastructure

.gitignore

@@ -11,18 +11,6 @@ setup.cfg
 */*.pyc
 *~
 
-bin/
-build/
-coverage/
-develop-eggs/
-dist/
-eggs/
-include/
-lib/
-local/
-parts/
-searx.egg-info/
-var/
 node_modules/
 
 .tx/

.travis.yml

@@ -11,16 +11,17 @@ before_install:
   - "export DISPLAY=:99.0"
   - "sh -e /etc/init.d/xvfb start"
   - npm install -g less grunt-cli
-  - ( cd searx/static/themes/oscar;npm install )
+  - ( cd searx/static/themes/oscar;npm install; cd - )
 install:
-  - "make"
+  - ./manage.sh update_packages
+  - ./manage.sh update_dev_packages
   - pip install coveralls
 script:
-  - "make flake8"
-  - "make robot"
-  - "make styles"
-  - "make grunt"
-  - make coverage
+  - ./manage.sh pep8_check
+  - ./manage.sh styles
+  - ./manage.sh grunt
+  - ./manage.sh coverage
+  - ./manage.sh robot
 after_success:
   coveralls
 notifications:

Makefile

-# convenience makefile to boostrap & run buildout
-# use `make options=-v` to run buildout with extra options
-
-version = 2.7
-python = bin/python
-options =
-
-all: .installed.cfg
-
-.installed.cfg: bin/buildout buildout.cfg setup.py
-	bin/buildout $(options)
-
-bin/buildout: $(python) buildout.cfg bootstrap.py
-	$(python) bootstrap.py
-	@touch $@
-
-$(python):
-	virtualenv -p python$(version) --no-site-packages .
-	@touch $@
-
-robot: .installed.cfg
-	@bin/robot
-
-flake8: .installed.cfg
-	@bin/flake8 setup.py
-	@bin/flake8 ./searx/
-
-tests: .installed.cfg flake8
-	@bin/test
-	@grunt test --gruntfile searx/static/themes/oscar/gruntfile.js
-
-coverage: .installed.cfg
-	@bin/coverage run bin/test
-	@bin/coverage report
-	@bin/coverage html
-
-production: bin/buildout production.cfg setup.py
-	bin/buildout -c production.cfg $(options)
-	@echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`"
-	@echo "* Hint 1: on production, disable debug mode and change secret_key"
-	@echo "* Hint 2: searx will be executed at server startup by crontab"
-	@echo "* Hint 3: to run immediatley, execute 'bin/supervisord'"
-
-minimal: bin/buildout minimal.cfg setup.py
-	bin/buildout -c minimal.cfg $(options)
-
-styles:
-	@lessc -x searx/static/themes/default/less/style.less > searx/static/themes/default/css/style.css
-	@lessc -x searx/static/themes/default/less/style-rtl.less > searx/static/themes/default/css/style-rtl.css
-	@lessc -x searx/static/themes/courgette/less/style.less > searx/static/themes/courgette/css/style.css
-	@lessc -x searx/static/themes/courgette/less/style-rtl.less > searx/static/themes/courgette/css/style-rtl.css
-	@lessc -x searx/static/less/bootstrap/bootstrap.less > searx/static/css/bootstrap.min.css
-	@lessc -x searx/static/themes/oscar/less/oscar/oscar.less > searx/static/themes/oscar/css/oscar.min.css
-	@lessc -x searx/static/themes/pix-art/less/style.less > searx/static/themes/pix-art/css/style.css
-
-grunt:
-	@grunt --gruntfile searx/static/themes/oscar/gruntfile.js
-
-locales:
-	@pybabel compile -d searx/translations
-
-clean:
-	@rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs eggs \
-		searx.egg-info lib include .coverage coverage
-
-.PHONY: all tests robot flake8 coverage production minimal styles locales clean

base.cfg

-[buildout]
-extends = versions.cfg
-unzip = true
-newest = false
-prefer-final = true
-develop = .
-
-eggs =
-    searx
-
-parts =
-    omelette
-
-
-[omelette]
-recipe = collective.recipe.omelette
-eggs = ${buildout:eggs}

bootstrap.py

-##############################################################################
-#
-# Copyright (c) 2006 Zope Foundation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-"""Bootstrap a buildout-based project
-
-Simply run this script in a directory containing a buildout.cfg.
-The script accepts buildout command-line options, so you can
-use the -c option to specify an alternate configuration file.
-"""
-
-import os
-import shutil
-import sys
-import tempfile
-
-from optparse import OptionParser
-
-__version__ = '2015-07-01'
-# See zc.buildout's changelog if this version is up to date.
-
-tmpeggs = tempfile.mkdtemp(prefix='bootstrap-')
-
-usage = '''\
-[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
-
-Bootstraps a buildout-based project.
-
-Simply run this script in a directory containing a buildout.cfg, using the
-Python that you want bin/buildout to use.
-
-Note that by using --find-links to point to local resources, you can keep
-this script from going over the network.
-'''
-
-parser = OptionParser(usage=usage)
-parser.add_option("--version",
-                  action="store_true", default=False,
-                  help=("Return bootstrap.py version."))
-parser.add_option("-t", "--accept-buildout-test-releases",
-                  dest='accept_buildout_test_releases',
-                  action="store_true", default=False,
-                  help=("Normally, if you do not specify a --version, the "
-                        "bootstrap script and buildout gets the newest "
-                        "*final* versions of zc.buildout and its recipes and "
-                        "extensions for you.  If you use this flag, "
-                        "bootstrap and buildout will get the newest releases "
-                        "even if they are alphas or betas."))
-parser.add_option("-c", "--config-file",
-                  help=("Specify the path to the buildout configuration "
-                        "file to be used."))
-parser.add_option("-f", "--find-links",
-                  help=("Specify a URL to search for buildout releases"))
-parser.add_option("--allow-site-packages",
-                  action="store_true", default=False,
-                  help=("Let bootstrap.py use existing site packages"))
-parser.add_option("--buildout-version",
-                  help="Use a specific zc.buildout version")
-parser.add_option("--setuptools-version",
-                  help="Use a specific setuptools version")
-parser.add_option("--setuptools-to-dir",
-                  help=("Allow for re-use of existing directory of "
-                        "setuptools versions"))
-
-options, args = parser.parse_args()
-if options.version:
-    print("bootstrap.py version %s" % __version__)
-    sys.exit(0)
-
-
-######################################################################
-# load/install setuptools
-
-try:
-    from urllib.request import urlopen
-except ImportError:
-    from urllib2 import urlopen
-
-ez = {}
-if os.path.exists('ez_setup.py'):
-    exec(open('ez_setup.py').read(), ez)
-else:
-    exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)
-
-if not options.allow_site_packages:
-    # ez_setup imports site, which adds site packages
-    # this will remove them from the path to ensure that incompatible versions
-    # of setuptools are not in the path
-    import site
-    # inside a virtualenv, there is no 'getsitepackages'.
-    # We can't remove these reliably
-    if hasattr(site, 'getsitepackages'):
-        for sitepackage_path in site.getsitepackages():
-            # Strip all site-packages directories from sys.path that
-            # are not sys.prefix; this is because on Windows
-            # sys.prefix is a site-package directory.
-            if sitepackage_path != sys.prefix:
-                sys.path[:] = [x for x in sys.path
-                               if sitepackage_path not in x]
-
-setup_args = dict(to_dir=tmpeggs, download_delay=0)
-
-if options.setuptools_version is not None:
-    setup_args['version'] = options.setuptools_version
-if options.setuptools_to_dir is not None:
-    setup_args['to_dir'] = options.setuptools_to_dir
-
-ez['use_setuptools'](**setup_args)
-import setuptools
-import pkg_resources
-
-# This does not (always?) update the default working set.  We will
-# do it.
-for path in sys.path:
-    if path not in pkg_resources.working_set.entries:
-        pkg_resources.working_set.add_entry(path)
-
-######################################################################
-# Install buildout
-
-ws = pkg_resources.working_set
-
-setuptools_path = ws.find(
-    pkg_resources.Requirement.parse('setuptools')).location
-
-# Fix sys.path here as easy_install.pth added before PYTHONPATH
-cmd = [sys.executable, '-c',
-       'import sys; sys.path[0:0] = [%r]; ' % setuptools_path +
-       'from setuptools.command.easy_install import main; main()',
-       '-mZqNxd', tmpeggs]
-
-find_links = os.environ.get(
-    'bootstrap-testing-find-links',
-    options.find_links or
-    ('http://downloads.buildout.org/'
-     if options.accept_buildout_test_releases else None)
-    )
-if find_links:
-    cmd.extend(['-f', find_links])
-
-requirement = 'zc.buildout'
-version = options.buildout_version
-if version is None and not options.accept_buildout_test_releases:
-    # Figure out the most recent final version of zc.buildout.
-    import setuptools.package_index
-    _final_parts = '*final-', '*final'
-
-    def _final_version(parsed_version):
-        try:
-            return not parsed_version.is_prerelease
-        except AttributeError:
-            # Older setuptools
-            for part in parsed_version:
-                if (part[:1] == '*') and (part not in _final_parts):
-                    return False
-            return True
-
-    index = setuptools.package_index.PackageIndex(
-        search_path=[setuptools_path])
-    if find_links:
-        index.add_find_links((find_links,))
-    req = pkg_resources.Requirement.parse(requirement)
-    if index.obtain(req) is not None:
-        best = []
-        bestv = None
-        for dist in index[req.project_name]:
-            distv = dist.parsed_version
-            if _final_version(distv):
-                if bestv is None or distv > bestv:
-                    best = [dist]
-                    bestv = distv
-                elif distv == bestv:
-                    best.append(dist)
-        if best:
-            best.sort()
-            version = best[-1].version
-if version:
-    requirement = '=='.join((requirement, version))
-cmd.append(requirement)
-
-import subprocess
-if subprocess.call(cmd) != 0:
-    raise Exception(
-        "Failed to execute command:\n%s" % repr(cmd)[1:-1])
-
-######################################################################
-# Import and run buildout
-
-ws.add_entry(tmpeggs)
-ws.require(requirement)
-import zc.buildout.buildout
-
-if not [a for a in args if '=' not in a]:
-    args.append('bootstrap')
-
-# if -c was provided, we push it back into args for buildout' main function
-if options.config_file is not None:
-    args[0:0] = ['-c', options.config_file]
-
-zc.buildout.buildout.main(args)
-shutil.rmtree(tmpeggs)

buildout.cfg

-[buildout]
-extends = base.cfg
-develop = .
-
-eggs =
-    searx [test]
-
-parts +=
-    pyscripts
-    robot
-    test
-
-
-[pyscripts]
-recipe = zc.recipe.egg:script
-eggs = ${buildout:eggs}
-interpreter = py
-dependent-scripts = true
-
-
-[robot]
-recipe = zc.recipe.testrunner
-eggs = ${buildout:eggs}
-defaults = ['--color', '--auto-progress', '--layer', 'SearxRobotLayer']
-
-
-[test]
-recipe = zc.recipe.testrunner
-eggs = ${buildout:eggs}
-defaults = ['--color', '--auto-progress', '--layer', 'SearxTestLayer', '--layer', '!SearxRobotLayer']

manage.sh

+#!/bin/sh
+
+BASE_DIR=$(dirname `readlink -f $0`)
+PYTHONPATH=$BASE_DIR
+SEARX_DIR="$BASE_DIR/searx"
+ACTION=$1
+
+update_packages() {
+    pip install --upgrade -r "$BASE_DIR/requirements.txt"
+}
+
+update_dev_packages() {
+    pip install --upgrade -r "$BASE_DIR/requirements-dev.txt"
+}
+
+pep8_check() {
+    echo '[!] Running pep8 check'
+    pep8 "$SEARX_DIR" "$BASE_DIR/tests"
+}
+
+unit_tests() {
+    echo '[!] Running unit tests'
+    python -m nose2 -s "$BASE_DIR/tests/unit"
+}
+
+py_test_coverage() {
+    echo '[!] Running python test coverage'
+    PYTHONPATH=`pwd` python -m nose2 -C --coverage "$SEARX_DIR" -s "$BASE_DIR/tests/unit"
+    coverage report
+    coverage html
+}
+
+robot_tests() {
+    echo '[!] Running robot tests'
+    python -c "import zope.testrunner; import sys; sys.exit(zope.testrunner.run("\
+"['--color', '--auto-progress', '--layer', 'SearxRobotLayer', '--test-path', '$BASE_DIR']))"
+}
+
+tests() {
+    set -e
+    pep8_check
+    unit_tests
+    robot_tests
+    set +e
+}
+
+build_style() {
+    lessc -x "$BASE_DIR/searx/static/$1" "$BASE_DIR/searx/static/$2"
+}
+
+styles() {
+    echo '[!] Building styles'
+	build_style themes/default/less/style.less themes/default/css/style.css
+	build_style themes/default/less/style-rtl.less themes/default/css/style-rtl.css
+	build_style themes/courgette/less/style.less themes/courgette/css/style.css
+	build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css
+	build_style less/bootstrap/bootstrap.less css/bootstrap.min.css
+	build_style themes/oscar/less/oscar/oscar.less themes/oscar/css/oscar.min.css
+	build_style themes/pix-art/less/style.less themes/pix-art/css/style.css
+}
+
+grunt() {
+	grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js"
+}
+
+locales() {
+	pybabel compile -d "$SEARX_DIR/translations"
+}
+
+$ACTION

minimal.cfg

-[buildout]
-extends = base.cfg
-develop = .
-
-eggs =
-    searx
-
-parts +=
-    pyscripts
-
-
-[pyscripts]
-recipe = zc.recipe.egg:script
-eggs = ${buildout:eggs}
-interpreter = py

production.cfg

-[buildout]
-extends = base.cfg
-develop = .
-
-eggs =
-    searx
-
-parts +=
-    pyscripts
-    supervisor
-    crontab_reboot
-
-
-[pyscripts]
-recipe = zc.recipe.egg:script
-eggs = ${buildout:eggs}
-interpreter = py
-
-
-[supervisor]
-recipe = collective.recipe.supervisor
-http-socket = unix
-user = searxer
-password = ohpleasedochangeme
-file = /tmp/supervisor.sock
-chmod = 0700
-programs =
-    50 searx    ${buildout:bin-directory}/searx-run
-
-
-[crontab_reboot]
-recipe = z3c.recipe.usercrontab
-times = @reboot
-command = ${buildout:bin-directory}/supervisord

requirements-dev.txt

+flake8==2.5.1
+mock==1.0.1
+nose2==0.4.7
+nose2[coverage-plugin]
+plone.testing==4.0.15
+robotframework-selenium2library==1.7.4
+robotsuite==1.7.0
+unittest2==1.1.0
+zope.testrunner==4.4.10

searx/tests/test_engines.py

-from searx.tests.engines.test_bing import *  # noqa
-from searx.tests.engines.test_bing_images import *  # noqa
-from searx.tests.engines.test_bing_news import *  # noqa
-from searx.tests.engines.test_blekko_images import *  # noqa
-from searx.tests.engines.test_btdigg import *  # noqa
-from searx.tests.engines.test_currency_convert import *  # noqa
-from searx.tests.engines.test_dailymotion import *  # noqa
-from searx.tests.engines.test_deezer import *  # noqa
-from searx.tests.engines.test_deviantart import *  # noqa
-from searx.tests.engines.test_digg import *  # noqa
-from searx.tests.engines.test_duckduckgo import *  # noqa
-from searx.tests.engines.test_duckduckgo_definitions import *  # noqa
-from searx.tests.engines.test_dummy import *  # noqa
-from searx.tests.engines.test_faroo import *  # noqa
-from searx.tests.engines.test_flickr import *  # noqa
-from searx.tests.engines.test_flickr_noapi import *  # noqa
-from searx.tests.engines.test_gigablast import *  # noqa
-from searx.tests.engines.test_github import *  # noqa
-from searx.tests.engines.test_google import *  # noqa
-from searx.tests.engines.test_google_images import *  # noqa
-from searx.tests.engines.test_google_news import *  # noqa
-from searx.tests.engines.test_kickass import *  # noqa
-from searx.tests.engines.test_mediawiki import *  # noqa
-from searx.tests.engines.test_mixcloud import *  # noqa
-from searx.tests.engines.test_openstreetmap import *  # noqa
-from searx.tests.engines.test_photon import *  # noqa
-from searx.tests.engines.test_piratebay import *  # noqa
-from searx.tests.engines.test_qwant import *  # noqa
-from searx.tests.engines.test_searchcode_code import *  # noqa
-from searx.tests.engines.test_searchcode_doc import *  # noqa
-from searx.tests.engines.test_soundcloud import *  # noqa
-from searx.tests.engines.test_spotify import *  # noqa
-from searx.tests.engines.test_stackoverflow import *  # noqa
-from searx.tests.engines.test_startpage import *  # noqa
-from searx.tests.engines.test_subtitleseeker import *  # noqa
-from searx.tests.engines.test_swisscows import *  # noqa
-from searx.tests.engines.test_twitter import *  # noqa
-from searx.tests.engines.test_vimeo import *  # noqa
-from searx.tests.engines.test_www1x import *  # noqa
-from searx.tests.engines.test_www500px import *  # noqa
-from searx.tests.engines.test_yacy import *  # noqa
-from searx.tests.engines.test_yahoo import *  # noqa
-from searx.tests.engines.test_youtube_api import *  # noqa
-from searx.tests.engines.test_youtube_noapi import *  # noqa
-from searx.tests.engines.test_yahoo_news import *  # noqa

versions.cfg

-[versions]
-Babel = 1.3
-Flask = 0.10.1
-Flask-Babel = 0.9
-Jinja2 = 2.7.3
-MarkupSafe = 0.23
-Pygments = 2.0.2
-WebOb = 1.4.1
-WebTest = 2.0.18
-Werkzeug = 0.10.4
-collective.recipe.omelette = 0.16
-coverage = 3.7.1
-decorator = 3.4.2
-docutils = 0.12
-flake8 = 2.4.1
-itsdangerous = 0.24
-mccabe = 0.3.1
-mock = 1.0.1
-pep8 = 1.5.7
-plone.testing = 4.0.13
-pyflakes = 0.8.1
-pytz = 2015.4
-pyyaml = 3.11
-requests = 2.7.0
-robotframework-debuglibrary = 0.3
-robotframework-httplibrary = 0.4.2
-robotframework-selenium2library = 1.7.1
-robotsuite = 1.6.1
-selenium = 2.46.0
-speaklater = 1.3
-unittest2 = 1.0.1
-waitress = 0.8.9
-zc.recipe.testrunner = 2.0.0
-pyopenssl = 0.15.1
-ndg-httpsclient = 0.4.0
-pyasn1 = 0.1.8
-pyasn1-modules = 0.0.6
-certifi = 2015.11.20.1
-
-cffi = 1.1.2
-cryptography = 0.9.1
-
-# Required by:
-# robotsuite==1.6.1
-# searx==0.7.0
-lxml = 3.4.4
-
-# Required by:
-# searx==0.7.0
-python-dateutil = 2.4.2
-
-# Required by:
-# searx==0.7.0
-# zope.exceptions==4.0.7
-# zope.interface==4.1.2
-# zope.testrunner==4.4.9
-setuptools = 18.0.1
-
-# Required by:
-# WebTest==2.0.18
-beautifulsoup4 = 4.3.2
-
-# Required by:
-# cryptography==0.9.1
-enum34 = 1.0.4
-
-# Required by:
-# cryptography==0.9.1
-idna = 2.0
-
-# Required by:
-# cryptography==0.9.1
-ipaddress = 1.0.7
-
-# Required by:
-# robotframework-httplibrary==0.4.2
-jsonpatch = 1.11
-
-# Required by:
-# robotframework-httplibrary==0.4.2
-jsonpointer = 1.9
-
-# Required by:
-# traceback2==1.4.0
-linecache2 = 1.0.0
-
-# Required by:
-# cffi==1.1.2
-pycparser = 2.12
-
-# Required by:
-# robotframework-httplibrary==0.4.2
-robotframework = 2.8.7
-
-# Required by:
-# robotsuite==1.6.1
-# zope.testrunner==4.4.9
-six = 1.9.0
-
-# Required by:
-# unittest2==1.0.1
-traceback2 = 1.4.0
-
-# Required by:
-# collective.recipe.omelette==0.16
-zc.recipe.egg = 2.0.1
-
-# Required by:
-# zope.testrunner==4.4.9
-zope.exceptions = 4.0.7
-
-# Required by:
-# zope.testrunner==4.4.9
-zope.interface = 4.1.2
-
-# Required by:
-# plone.testing==4.0.13
-zope.testing = 4.2.0
-
-# Required by:
-# zc.recipe.testrunner==2.0.0
-zope.testrunner = 4.4.9