fix(pyspark): avoid potentially different field names produced by SQL by using python-native APIs (#10877)

Author: cpcloud

ibis/backends/pyspark/__init__.py

@@ -356,9 +356,7 @@ def list_databases(
         self, *, like: str | None = None, catalog: str | None = None
     ) -> list[str]:
         with self._active_catalog(catalog):
-            databases = [
-                db.namespace for db in self._session.sql("SHOW DATABASES").collect()
-            ]
+            databases = [db.name for db in self._session.catalog.listDatabases()]
         return self._filter_with_like(databases, like)
 
     def list_tables(

ibis/backends/pyspark/tests/test_client.py

@@ -39,6 +39,10 @@ def test_catalog_db_args(con):
     assert con.current_database == "ibis_testing"
 
 
+def test_list_databases(con):
+    assert "ibis_testing" in con.list_databases(catalog="spark_catalog")
+
+
 def test_create_table_no_catalog(con):
     t = ibis.memtable({"epoch": [1712848119, 1712848121, 1712848155]})