diff --git a/.travis.yml b/.travis.yml index 899286ce99290..42ca4116fce6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: include: - dist: trusty env: - - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network" + - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network and not db" - dist: trusty env: diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index b9e0cd0b9258c..c0c4fb924a605 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -12,37 +12,37 @@ jobs: py35_np_120: ENV_FILE: ci/deps/azure-macos-35.yaml CONDA_PY: "35" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not db" ${{ if eq(parameters.name, 'Linux') }}: py27_np_120: ENV_FILE: ci/deps/azure-27-compat.yaml CONDA_PY: "27" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not db" py27_locale_slow_old_np: ENV_FILE: ci/deps/azure-27-locale.yaml CONDA_PY: "27" - PATTERN: "slow" + PATTERN: "slow and not db" LOCALE_OVERRIDE: "zh_CN.UTF-8" EXTRA_APT: "language-pack-zh-hans" py36_locale_slow: ENV_FILE: ci/deps/azure-36-locale_slow.yaml CONDA_PY: "36" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not db" LOCALE_OVERRIDE: "it_IT.UTF-8" py37_locale: ENV_FILE: ci/deps/azure-37-locale.yaml CONDA_PY: "37" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not db" LOCALE_OVERRIDE: "zh_CN.UTF-8" py37_np_dev: ENV_FILE: ci/deps/azure-37-numpydev.yaml CONDA_PY: "37" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not db" TEST_ARGS: "-W error" PANDAS_TESTING_MODE: "deprecate" EXTRA_APT: "xsel" diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index cece002024936..f06b229bb2656 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -38,7 +38,7 @@ jobs: displayName: 'Build' - script: | call activate pandas-dev - pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %* + pytest -m "not slow and not network and not db" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %* displayName: 'Test' - task: PublishTestResults@2 inputs: diff --git a/ci/deps/azure-27-compat.yaml b/ci/deps/azure-27-compat.yaml index f3cc615c35243..8899e22bdf6cf 100644 --- a/ci/deps/azure-27-compat.yaml +++ b/ci/deps/azure-27-compat.yaml @@ -9,13 +9,11 @@ dependencies: - numexpr=2.6.1 - numpy=1.12.0 - openpyxl=2.5.5 - - psycopg2 - pytables=3.4.2 - python-dateutil=2.5.0 - python=2.7* - pytz=2013b - scipy=0.18.1 - - sqlalchemy=0.7.8 - xlrd=1.0.0 - xlsxwriter=0.5.2 - xlwt=0.7.5 @@ -25,5 +23,4 @@ dependencies: - pip: - html5lib==1.0b2 - beautifulsoup4==4.2.1 - - pymysql==0.6.0 - hypothesis>=3.58.0 diff --git a/ci/deps/azure-36-locale_slow.yaml b/ci/deps/azure-36-locale_slow.yaml index 4bbc6a2c11f1e..c7d2334623501 100644 --- a/ci/deps/azure-36-locale_slow.yaml +++ b/ci/deps/azure-36-locale_slow.yaml @@ -15,15 +15,12 @@ dependencies: - numexpr - numpy - openpyxl - - psycopg2 - - pymysql - pytables - python-dateutil - python=3.6* - pytz - s3fs - scipy - - sqlalchemy - xarray - xlrd - xlsxwriter diff --git a/ci/deps/azure-37-locale.yaml b/ci/deps/azure-37-locale.yaml index 11a698ce7648e..b5a05c49b8083 100644 --- a/ci/deps/azure-37-locale.yaml +++ b/ci/deps/azure-37-locale.yaml @@ -14,15 +14,12 @@ dependencies: - numexpr - numpy - openpyxl - - psycopg2 - - pymysql - pytables - python-dateutil - python=3.7* - pytz - s3fs - scipy - - sqlalchemy - xarray - xlrd - xlsxwriter diff --git a/pandas/conftest.py b/pandas/conftest.py index c72efdf865052..5aa644bb942f8 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -34,6 +34,8 @@ def pytest_addoption(parser): help="skip slow tests") parser.addoption("--skip-network", action="store_true", help="skip network tests") + parser.addoption("--skip-db", action="store_true", + help="skip db tests") parser.addoption("--run-high-memory", action="store_true", help="run high memory tests") parser.addoption("--only-slow", action="store_true", @@ -52,6 +54,9 @@ def pytest_runtest_setup(item): if 'network' in item.keywords and item.config.getoption("--skip-network"): pytest.skip("skipping due to --skip-network") + if 'db' in item.keywords and item.config.getoption("--skip-db"): + pytest.skip("skipping due to --skip-db") + if 'high_memory' in item.keywords and not item.config.getoption( "--run-high-memory"): pytest.skip( diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index eeeb55cb8e70c..8f2a212b457a4 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1152,14 +1152,8 @@ class _TestSQLAlchemy(SQLAlchemyMixIn, PandasSQLTest): def setup_class(cls): cls.setup_import() cls.setup_driver() - - # test connection - try: - conn = cls.connect() - conn.connect() - except sqlalchemy.exc.OperationalError: - msg = "{0} - can't connect to {1} server".format(cls, cls.flavor) - pytest.skip(msg) + conn = cls.connect() + conn.connect() def load_test_data_and_sql(self): self._load_raw_sql() @@ -1791,13 +1785,10 @@ def connect(cls): @classmethod def setup_driver(cls): - try: - import pymysql # noqa - cls.driver = 'pymysql' - from pymysql.constants import CLIENT - cls.connect_args = {'client_flag': CLIENT.MULTI_STATEMENTS} - except ImportError: - pytest.skip('pymysql not installed') + pymysql = pytest.importorskip('pymysql') + cls.driver = 'pymysql' + cls.connect_args = { + 'client_flag': pymysql.constants.CLIENT.MULTI_STATEMENTS} def test_default_type_conversion(self): df = sql.read_sql_table("types_test_data", self.conn) @@ -1860,11 +1851,8 @@ def connect(cls): @classmethod def setup_driver(cls): - try: - import psycopg2 # noqa - cls.driver = 'psycopg2' - except ImportError: - pytest.skip('psycopg2 not installed') + pytest.importorskip('psycopg2') + cls.driver = 'psycopg2' def test_schema_support(self): # only test this for postgresql (schema's not supported in @@ -1932,21 +1920,25 @@ def test_schema_support(self): @pytest.mark.single +@pytest.mark.db class TestMySQLAlchemy(_TestMySQLAlchemy, _TestSQLAlchemy): pass @pytest.mark.single +@pytest.mark.db class TestMySQLAlchemyConn(_TestMySQLAlchemy, _TestSQLAlchemyConn): pass @pytest.mark.single +@pytest.mark.db class TestPostgreSQLAlchemy(_TestPostgreSQLAlchemy, _TestSQLAlchemy): pass @pytest.mark.single +@pytest.mark.db class TestPostgreSQLAlchemyConn(_TestPostgreSQLAlchemy, _TestSQLAlchemyConn): pass @@ -2189,13 +2181,6 @@ def tquery(query, con=None, cur=None): return list(res) -def _skip_if_no_pymysql(): - try: - import pymysql # noqa - except ImportError: - pytest.skip('pymysql not installed, skipping') - - @pytest.mark.single class TestXSQLite(SQLiteMixIn): @@ -2404,76 +2389,56 @@ def clean_up(test_table_to_drop): @pytest.mark.single +@pytest.mark.db @pytest.mark.skip(reason="gh-13611: there is no support for MySQL " "if SQLAlchemy is not installed") class TestXMySQL(MySQLMixIn): @pytest.fixture(autouse=True, scope='class') def setup_class(cls): - _skip_if_no_pymysql() - - # test connection - import pymysql - try: - # Try Travis defaults. - # No real user should allow root access with a blank password. - pymysql.connect(host='localhost', user='root', passwd='', - db='pandas_nosetest') - except pymysql.Error: - pass - else: - return + pymysql = pytest.importorskip('pymysql') + pymysql.connect(host='localhost', user='root', passwd='', + db='pandas_nosetest') try: pymysql.connect(read_default_group='pandas') except pymysql.ProgrammingError: - pytest.skip( + raise RuntimeError( "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " - "typically located at ~/.my.cnf or /etc/.my.cnf. ") + "typically located at ~/.my.cnf or /etc/.my.cnf.") except pymysql.Error: - pytest.skip( + raise RuntimeError( "Cannot connect to database. " "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " - "typically located at ~/.my.cnf or /etc/.my.cnf. ") + "typically located at ~/.my.cnf or /etc/.my.cnf.") @pytest.fixture(autouse=True) def setup_method(self, request, datapath): - _skip_if_no_pymysql() - import pymysql + pymysql = pytest.importorskip('pymysql') + pymysql.connect(host='localhost', user='root', passwd='', + db='pandas_nosetest') try: - # Try Travis defaults. - # No real user should allow root access with a blank password. - self.conn = pymysql.connect(host='localhost', user='root', - passwd='', db='pandas_nosetest') - except pymysql.Error: - pass - else: - return - try: - self.conn = pymysql.connect(read_default_group='pandas') + pymysql.connect(read_default_group='pandas') except pymysql.ProgrammingError: - pytest.skip( + raise RuntimeError( "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " - "typically located at ~/.my.cnf or /etc/.my.cnf. ") + "typically located at ~/.my.cnf or /etc/.my.cnf.") except pymysql.Error: - pytest.skip( + raise RuntimeError( "Cannot connect to database. " "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " - "typically located at ~/.my.cnf or /etc/.my.cnf. ") + "typically located at ~/.my.cnf or /etc/.my.cnf.") self.method = request.function def test_basic(self): - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() self._check_roundtrip(frame) def test_write_row_by_row(self): - - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() frame.iloc[0, 0] = np.nan drop_sql = "DROP TABLE IF EXISTS test" @@ -2493,7 +2458,6 @@ def test_write_row_by_row(self): tm.assert_frame_equal(result, frame, check_less_precise=True) def test_chunksize_read_type(self): - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() frame.index.name = "index" drop_sql = "DROP TABLE IF EXISTS test" @@ -2508,7 +2472,6 @@ def test_chunksize_read_type(self): tm.assert_frame_equal(frame[:chunksize], chunk_df) def test_execute(self): - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() drop_sql = "DROP TABLE IF EXISTS test" create_sql = sql.get_schema(frame, 'test') @@ -2528,7 +2491,6 @@ def test_execute(self): tm.assert_frame_equal(result, frame[:1]) def test_schema(self): - _skip_if_no_pymysql() frame = tm.makeTimeDataFrame() create_sql = sql.get_schema(frame, 'test') lines = create_sql.splitlines() @@ -2548,7 +2510,6 @@ def test_schema(self): @tm.capture_stdout def test_execute_fail(self): - _skip_if_no_pymysql() drop_sql = "DROP TABLE IF EXISTS test" create_sql = """ CREATE TABLE test @@ -2570,7 +2531,6 @@ def test_execute_fail(self): sql.execute('INSERT INTO test VALUES("foo", "bar", 7)', self.conn) def test_execute_closed_connection(self, request, datapath): - _skip_if_no_pymysql() drop_sql = "DROP TABLE IF EXISTS test" create_sql = """ CREATE TABLE test @@ -2595,11 +2555,9 @@ def test_execute_closed_connection(self, request, datapath): self.setup_method(request, datapath) def test_na_roundtrip(self): - _skip_if_no_pymysql() pass def _check_roundtrip(self, frame): - _skip_if_no_pymysql() drop_sql = "DROP TABLE IF EXISTS test_table" cur = self.conn.cursor() with warnings.catch_warnings(): @@ -2636,13 +2594,11 @@ def _check_roundtrip(self, frame): tm.assert_frame_equal(expected, result) def test_keyword_as_column_names(self): - _skip_if_no_pymysql() df = DataFrame({'From': np.ones(5)}) sql.to_sql(df, con=self.conn, name='testkeywords', if_exists='replace', index=False) def test_if_exists(self): - _skip_if_no_pymysql() df_if_exists_1 = DataFrame({'col1': [1, 2], 'col2': ['A', 'B']}) df_if_exists_2 = DataFrame( {'col1': [3, 4, 5], 'col2': ['C', 'D', 'E']}) diff --git a/pandas/util/_tester.py b/pandas/util/_tester.py index aad2f00fa0478..18e8d415459fd 100644 --- a/pandas/util/_tester.py +++ b/pandas/util/_tester.py @@ -16,7 +16,7 @@ def test(extra_args=None): import hypothesis # noqa except ImportError: raise ImportError("Need hypothesis>=3.58 to run tests") - cmd = ['--skip-slow', '--skip-network'] + cmd = ['--skip-slow', '--skip-network', '--skip-db'] if extra_args: if not isinstance(extra_args, list): extra_args = [extra_args] diff --git a/setup.cfg b/setup.cfg index c199567737531..fceff01f0671f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,6 +67,7 @@ markers = single: mark a test as single cpu only slow: mark a test as slow network: mark a test as network + db: tests requiring a database (mysql or postgres) high_memory: mark a test as a high-memory only clipboard: mark a pd.read_clipboard test doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL