From 432e107bc6a2dda19385b7c423d2768e3a879965 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Thu, 16 May 2024 14:13:05 +0200 Subject: [PATCH 01/48] CASSANDRA-19635: Run integration tests with C* 5.x patch by Lukasz Antoniak; reviewed by Andy Tolbert, and Bret McGuire for CASSANDRA-19635 --- integration-tests/pom.xml | 3 + .../core/auth/DseProxyAuthenticationIT.java | 60 ++++++++------- .../oss/driver/core/cql/AsyncResultSetIT.java | 18 +++-- .../oss/driver/core/cql/BatchStatementIT.java | 18 +++-- .../driver/core/cql/BoundStatementCcmIT.java | 73 ++++++++++--------- .../core/cql/ExecutionInfoWarningsIT.java | 17 +++-- .../oss/driver/core/cql/PagingStateIT.java | 14 ++-- .../driver/core/cql/PerRequestKeyspaceIT.java | 60 ++++++++------- .../core/cql/PreparedStatementCachingIT.java | 49 ++++++++++--- .../reactive/DefaultReactiveResultSetIT.java | 32 ++++---- .../oss/driver/core/metadata/DescribeIT.java | 23 +++--- .../oss/driver/core/metadata/SchemaIT.java | 14 ++++ .../type/codec/registry/CodecRegistryIT.java | 63 ++++++++-------- .../datastax/oss/driver/mapper/DeleteIT.java | 9 +-- .../oss/driver/mapper/DeleteReactiveIT.java | 18 +++-- .../driver/mapper/EntityPolymorphismIT.java | 38 ++++++---- .../oss/driver/mapper/ImmutableEntityIT.java | 14 +++- .../oss/driver/mapper/InventoryITBase.java | 8 +- .../oss/driver/mapper/NestedUdtIT.java | 48 ++++++------ .../mapper/SelectCustomWhereClauseIT.java | 14 +++- .../oss/driver/mapper/SelectReactiveIT.java | 14 +++- .../datastax/oss/driver/mapper/UpdateIT.java | 21 ++++-- .../osgi/support/CcmStagedReactor.java | 2 +- pom.xml | 11 +++ .../driver/api/testinfra/ccm/BaseCcmRule.java | 4 +- .../driver/api/testinfra/ccm/CcmBridge.java | 20 +++-- .../ccm/SchemaChangeSynchronizer.java | 42 +++++++++++ .../api/testinfra/session/SessionRule.java | 6 +- 28 files changed, 455 insertions(+), 258 deletions(-) create mode 100644 test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/SchemaChangeSynchronizer.java diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 32cabdb34a7..d1b0a736bb0 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -242,6 +242,8 @@ 8 ${project.build.directory}/failsafe-reports/failsafe-summary-parallelized.xml ${skipParallelizableITs} + ${blockhound.argline} + ${testing.jvm}/bin/java @@ -253,6 +255,7 @@ com.datastax.oss.driver.categories.ParallelizableTests, com.datastax.oss.driver.categories.IsolatedTests ${project.build.directory}/failsafe-reports/failsafe-summary-serial.xml ${skipSerialITs} + ${blockhound.argline} ${testing.jvm}/bin/java diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/auth/DseProxyAuthenticationIT.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/auth/DseProxyAuthenticationIT.java index 126a110da1a..a3f1c04afc0 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/auth/DseProxyAuthenticationIT.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/auth/DseProxyAuthenticationIT.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.cql.ResultSet; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.servererrors.UnauthorizedException; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; @@ -57,33 +58,38 @@ public static void addUsers() { @Before public void setupRoles() { - try (CqlSession session = ads.newKeyTabSession()) { - session.execute( - "CREATE ROLE IF NOT EXISTS alice WITH PASSWORD = 'fakePasswordForAlice' AND LOGIN = FALSE"); - session.execute( - "CREATE ROLE IF NOT EXISTS ben WITH PASSWORD = 'fakePasswordForBen' AND LOGIN = TRUE"); - session.execute("CREATE ROLE IF NOT EXISTS 'bob@DATASTAX.COM' WITH LOGIN = TRUE"); - session.execute( - "CREATE ROLE IF NOT EXISTS 'charlie@DATASTAX.COM' WITH PASSWORD = 'fakePasswordForCharlie' AND LOGIN = TRUE"); - session.execute( - "CREATE ROLE IF NOT EXISTS steve WITH PASSWORD = 'fakePasswordForSteve' AND LOGIN = TRUE"); - session.execute( - "CREATE KEYSPACE IF NOT EXISTS aliceks WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'}"); - session.execute( - "CREATE TABLE IF NOT EXISTS aliceks.alicetable (key text PRIMARY KEY, value text)"); - session.execute("INSERT INTO aliceks.alicetable (key, value) VALUES ('hello', 'world')"); - session.execute("GRANT ALL ON KEYSPACE aliceks TO alice"); - session.execute("GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'ben'"); - session.execute("GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'bob@DATASTAX.COM'"); - session.execute("GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'steve'"); - session.execute("GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'charlie@DATASTAX.COM'"); - session.execute("GRANT PROXY.LOGIN ON ROLE 'alice' TO 'ben'"); - session.execute("GRANT PROXY.LOGIN ON ROLE 'alice' TO 'bob@DATASTAX.COM'"); - session.execute("GRANT PROXY.EXECUTE ON ROLE 'alice' TO 'steve'"); - session.execute("GRANT PROXY.EXECUTE ON ROLE 'alice' TO 'charlie@DATASTAX.COM'"); - // ben and bob are allowed to login as alice, but not execute as alice. - // charlie and steve are allowed to execute as alice, but not login as alice. - } + SchemaChangeSynchronizer.withLock( + () -> { + try (CqlSession session = ads.newKeyTabSession()) { + session.execute( + "CREATE ROLE IF NOT EXISTS alice WITH PASSWORD = 'fakePasswordForAlice' AND LOGIN = FALSE"); + session.execute( + "CREATE ROLE IF NOT EXISTS ben WITH PASSWORD = 'fakePasswordForBen' AND LOGIN = TRUE"); + session.execute("CREATE ROLE IF NOT EXISTS 'bob@DATASTAX.COM' WITH LOGIN = TRUE"); + session.execute( + "CREATE ROLE IF NOT EXISTS 'charlie@DATASTAX.COM' WITH PASSWORD = 'fakePasswordForCharlie' AND LOGIN = TRUE"); + session.execute( + "CREATE ROLE IF NOT EXISTS steve WITH PASSWORD = 'fakePasswordForSteve' AND LOGIN = TRUE"); + session.execute( + "CREATE KEYSPACE IF NOT EXISTS aliceks WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'}"); + session.execute( + "CREATE TABLE IF NOT EXISTS aliceks.alicetable (key text PRIMARY KEY, value text)"); + session.execute( + "INSERT INTO aliceks.alicetable (key, value) VALUES ('hello', 'world')"); + session.execute("GRANT ALL ON KEYSPACE aliceks TO alice"); + session.execute("GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'ben'"); + session.execute("GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'bob@DATASTAX.COM'"); + session.execute("GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'steve'"); + session.execute( + "GRANT EXECUTE ON ALL AUTHENTICATION SCHEMES TO 'charlie@DATASTAX.COM'"); + session.execute("GRANT PROXY.LOGIN ON ROLE 'alice' TO 'ben'"); + session.execute("GRANT PROXY.LOGIN ON ROLE 'alice' TO 'bob@DATASTAX.COM'"); + session.execute("GRANT PROXY.EXECUTE ON ROLE 'alice' TO 'steve'"); + session.execute("GRANT PROXY.EXECUTE ON ROLE 'alice' TO 'charlie@DATASTAX.COM'"); + // ben and bob are allowed to login as alice, but not execute as alice. + // charlie and steve are allowed to execute as alice, but not login as alice. + } + }); } /** * Validates that a connection may be successfully made as user 'alice' using the credentials of a diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/AsyncResultSetIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/AsyncResultSetIT.java index 2d01043b46a..e109c28525e 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/AsyncResultSetIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/AsyncResultSetIT.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.ParallelizableTests; @@ -67,13 +68,16 @@ public class AsyncResultSetIT { @BeforeClass public static void setupSchema() { // create table and load data across two partitions so we can test paging across tokens. - SESSION_RULE - .session() - .execute( - SimpleStatement.builder( - "CREATE TABLE IF NOT EXISTS test (k0 text, k1 int, v int, PRIMARY KEY(k0, k1))") - .setExecutionProfile(SESSION_RULE.slowProfile()) - .build()); + SchemaChangeSynchronizer.withLock( + () -> { + SESSION_RULE + .session() + .execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS test (k0 text, k1 int, v int, PRIMARY KEY(k0, k1))") + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + }); PreparedStatement prepared = SESSION_RULE.session().prepare("INSERT INTO test (k0, k1, v) VALUES (?, ?, ?)"); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BatchStatementIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BatchStatementIT.java index 04e5798be5a..8b652638729 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BatchStatementIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BatchStatementIT.java @@ -34,6 +34,7 @@ import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.core.servererrors.InvalidQueryException; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; @@ -72,13 +73,16 @@ public void createTable() { "CREATE TABLE counter3 (k0 text PRIMARY KEY, c counter)", }; - for (String schemaStatement : schemaStatements) { - sessionRule - .session() - .execute( - SimpleStatement.newInstance(schemaStatement) - .setExecutionProfile(sessionRule.slowProfile())); - } + SchemaChangeSynchronizer.withLock( + () -> { + for (String schemaStatement : schemaStatements) { + sessionRule + .session() + .execute( + SimpleStatement.newInstance(schemaStatement) + .setExecutionProfile(sessionRule.slowProfile())); + } + }); } @Test diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BoundStatementCcmIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BoundStatementCcmIT.java index 79156fcce50..9e4b62cd230 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BoundStatementCcmIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BoundStatementCcmIT.java @@ -40,6 +40,7 @@ import com.datastax.oss.driver.api.core.metadata.token.Token; import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; @@ -94,40 +95,44 @@ public class BoundStatementCcmIT { @Before public void setupSchema() { // table where every column forms the primary key. - sessionRule - .session() - .execute( - SimpleStatement.builder( - "CREATE TABLE IF NOT EXISTS test (k text, v int, PRIMARY KEY(k, v))") - .setExecutionProfile(sessionRule.slowProfile()) - .build()); - for (int i = 0; i < 100; i++) { - sessionRule - .session() - .execute( - SimpleStatement.builder("INSERT INTO test (k, v) VALUES (?, ?)") - .addPositionalValues(KEY, i) - .build()); - } - - // table with simple primary key, single cell. - sessionRule - .session() - .execute( - SimpleStatement.builder("CREATE TABLE IF NOT EXISTS test2 (k text primary key, v0 int)") - .setExecutionProfile(sessionRule.slowProfile()) - .build()); - - // table with composite partition key - sessionRule - .session() - .execute( - SimpleStatement.builder( - "CREATE TABLE IF NOT EXISTS test3 " - + "(pk1 int, pk2 int, v int, " - + "PRIMARY KEY ((pk1, pk2)))") - .setExecutionProfile(sessionRule.slowProfile()) - .build()); + SchemaChangeSynchronizer.withLock( + () -> { + sessionRule + .session() + .execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS test (k text, v int, PRIMARY KEY(k, v))") + .setExecutionProfile(sessionRule.slowProfile()) + .build()); + for (int i = 0; i < 100; i++) { + sessionRule + .session() + .execute( + SimpleStatement.builder("INSERT INTO test (k, v) VALUES (?, ?)") + .addPositionalValues(KEY, i) + .build()); + } + + // table with simple primary key, single cell. + sessionRule + .session() + .execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS test2 (k text primary key, v0 int)") + .setExecutionProfile(sessionRule.slowProfile()) + .build()); + + // table with composite partition key + sessionRule + .session() + .execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS test3 " + + "(pk1 int, pk2 int, v int, " + + "PRIMARY KEY ((pk1, pk2)))") + .setExecutionProfile(sessionRule.slowProfile()) + .build()); + }); } @Test diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/ExecutionInfoWarningsIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/ExecutionInfoWarningsIT.java index 5907206d11a..edee9723a38 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/ExecutionInfoWarningsIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/ExecutionInfoWarningsIT.java @@ -33,6 +33,7 @@ import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; @@ -88,12 +89,16 @@ public class ExecutionInfoWarningsIT { @Before public void createSchema() { // table with simple primary key, single cell. - sessionRule - .session() - .execute( - SimpleStatement.builder("CREATE TABLE IF NOT EXISTS test (k int primary key, v text)") - .setExecutionProfile(sessionRule.slowProfile()) - .build()); + SchemaChangeSynchronizer.withLock( + () -> { + sessionRule + .session() + .execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS test (k int primary key, v text)") + .setExecutionProfile(sessionRule.slowProfile()) + .build()); + }); for (int i = 0; i < 100; i++) { sessionRule .session() diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PagingStateIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PagingStateIT.java index dcd801f19a4..6d33f35238a 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PagingStateIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PagingStateIT.java @@ -30,6 +30,7 @@ import com.datastax.oss.driver.api.core.type.codec.MappingCodec; import com.datastax.oss.driver.api.core.type.reflect.GenericType; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.ParallelizableTests; @@ -55,11 +56,14 @@ public class PagingStateIT { @Before public void setupSchema() { CqlSession session = SESSION_RULE.session(); - session.execute( - SimpleStatement.builder( - "CREATE TABLE IF NOT EXISTS foo (k int, cc int, v int, PRIMARY KEY(k, cc))") - .setExecutionProfile(SESSION_RULE.slowProfile()) - .build()); + SchemaChangeSynchronizer.withLock( + () -> { + session.execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS foo (k int, cc int, v int, PRIMARY KEY(k, cc))") + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + }); for (int i = 0; i < 20; i++) { session.execute( SimpleStatement.newInstance("INSERT INTO foo (k, cc, v) VALUES (1, ?, ?)", i, i)); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PerRequestKeyspaceIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PerRequestKeyspaceIT.java index 2b418e76f75..9eb883144db 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PerRequestKeyspaceIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PerRequestKeyspaceIT.java @@ -31,6 +31,7 @@ import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; @@ -67,13 +68,16 @@ public class PerRequestKeyspaceIT { @Before public void setupSchema() { - sessionRule - .session() - .execute( - SimpleStatement.builder( - "CREATE TABLE IF NOT EXISTS foo (k text, cc int, v int, PRIMARY KEY(k, cc))") - .setExecutionProfile(sessionRule.slowProfile()) - .build()); + SchemaChangeSynchronizer.withLock( + () -> { + sessionRule + .session() + .execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS foo (k text, cc int, v int, PRIMARY KEY(k, cc))") + .setExecutionProfile(sessionRule.slowProfile()) + .build()); + }); } @Test @@ -220,27 +224,31 @@ public void should_prepare_statement_with_keyspace() { @BackendRequirement(type = BackendType.CASSANDRA, minInclusive = "4.0") public void should_reprepare_statement_with_keyspace_on_the_fly() { // Create a separate session because we don't want it to have a default keyspace - try (CqlSession session = SessionUtils.newSession(ccmRule)) { - executeDdl( - session, - String.format( - "CREATE TABLE IF NOT EXISTS %s.bar (k int primary key)", sessionRule.keyspace())); - PreparedStatement pst = - session.prepare( - SimpleStatement.newInstance("SELECT * FROM bar WHERE k=?") - .setKeyspace(sessionRule.keyspace())); + SchemaChangeSynchronizer.withLock( + () -> { + try (CqlSession session = SessionUtils.newSession(ccmRule)) { + executeDdl( + session, + String.format( + "CREATE TABLE IF NOT EXISTS %s.bar (k int primary key)", + sessionRule.keyspace())); + PreparedStatement pst = + session.prepare( + SimpleStatement.newInstance("SELECT * FROM bar WHERE k=?") + .setKeyspace(sessionRule.keyspace())); - // Drop and re-create the table to invalidate the prepared statement server side - executeDdl(session, String.format("DROP TABLE %s.bar", sessionRule.keyspace())); - executeDdl( - session, - String.format("CREATE TABLE %s.bar (k int primary key)", sessionRule.keyspace())); - assertThat(preparedStatementExistsOnServer(session, pst.getId())).isFalse(); + // Drop and re-create the table to invalidate the prepared statement server side + executeDdl(session, String.format("DROP TABLE %s.bar", sessionRule.keyspace())); + executeDdl( + session, + String.format("CREATE TABLE %s.bar (k int primary key)", sessionRule.keyspace())); + assertThat(preparedStatementExistsOnServer(session, pst.getId())).isFalse(); - // This will re-prepare on the fly - session.execute(pst.bind(0)); - assertThat(preparedStatementExistsOnServer(session, pst.getId())).isTrue(); - } + // This will re-prepare on the fly + session.execute(pst.bind(0)); + assertThat(preparedStatementExistsOnServer(session, pst.getId())).isTrue(); + } + }); } private void executeDdl(CqlSession session, String query) { diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java index 92c6fd8a12e..05ac3bd0e92 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java @@ -30,6 +30,7 @@ import com.datastax.oss.driver.api.core.session.ProgrammaticArguments; import com.datastax.oss.driver.api.core.session.SessionBuilder; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.IsolatedTests; @@ -305,12 +306,19 @@ private void invalidationTestInner( @Test public void should_invalidate_cache_entry_on_basic_udt_change_result_set() { - invalidationResultSetTest(setupCacheEntryTestBasic, ImmutableSet.of("test_type_2")); + SchemaChangeSynchronizer.withLock( + () -> { + invalidationResultSetTest(setupCacheEntryTestBasic, ImmutableSet.of("test_type_2")); + }); } @Test public void should_invalidate_cache_entry_on_basic_udt_change_variable_defs() { - invalidationVariableDefsTest(setupCacheEntryTestBasic, false, ImmutableSet.of("test_type_2")); + SchemaChangeSynchronizer.withLock( + () -> { + invalidationVariableDefsTest( + setupCacheEntryTestBasic, false, ImmutableSet.of("test_type_2")); + }); } Consumer setupCacheEntryTestCollection = @@ -325,13 +333,19 @@ public void should_invalidate_cache_entry_on_basic_udt_change_variable_defs() { @Test public void should_invalidate_cache_entry_on_collection_udt_change_result_set() { - invalidationResultSetTest(setupCacheEntryTestCollection, ImmutableSet.of("test_type_2")); + SchemaChangeSynchronizer.withLock( + () -> { + invalidationResultSetTest(setupCacheEntryTestCollection, ImmutableSet.of("test_type_2")); + }); } @Test public void should_invalidate_cache_entry_on_collection_udt_change_variable_defs() { - invalidationVariableDefsTest( - setupCacheEntryTestCollection, true, ImmutableSet.of("test_type_2")); + SchemaChangeSynchronizer.withLock( + () -> { + invalidationVariableDefsTest( + setupCacheEntryTestCollection, true, ImmutableSet.of("test_type_2")); + }); } Consumer setupCacheEntryTestTuple = @@ -346,12 +360,19 @@ public void should_invalidate_cache_entry_on_collection_udt_change_variable_defs @Test public void should_invalidate_cache_entry_on_tuple_udt_change_result_set() { - invalidationResultSetTest(setupCacheEntryTestTuple, ImmutableSet.of("test_type_2")); + SchemaChangeSynchronizer.withLock( + () -> { + invalidationResultSetTest(setupCacheEntryTestTuple, ImmutableSet.of("test_type_2")); + }); } @Test public void should_invalidate_cache_entry_on_tuple_udt_change_variable_defs() { - invalidationVariableDefsTest(setupCacheEntryTestTuple, false, ImmutableSet.of("test_type_2")); + SchemaChangeSynchronizer.withLock( + () -> { + invalidationVariableDefsTest( + setupCacheEntryTestTuple, false, ImmutableSet.of("test_type_2")); + }); } Consumer setupCacheEntryTestNested = @@ -366,14 +387,20 @@ public void should_invalidate_cache_entry_on_tuple_udt_change_variable_defs() { @Test public void should_invalidate_cache_entry_on_nested_udt_change_result_set() { - invalidationResultSetTest( - setupCacheEntryTestNested, ImmutableSet.of("test_type_2", "test_type_4")); + SchemaChangeSynchronizer.withLock( + () -> { + invalidationResultSetTest( + setupCacheEntryTestNested, ImmutableSet.of("test_type_2", "test_type_4")); + }); } @Test public void should_invalidate_cache_entry_on_nested_udt_change_variable_defs() { - invalidationVariableDefsTest( - setupCacheEntryTestNested, false, ImmutableSet.of("test_type_2", "test_type_4")); + SchemaChangeSynchronizer.withLock( + () -> { + invalidationVariableDefsTest( + setupCacheEntryTestNested, false, ImmutableSet.of("test_type_2", "test_type_4")); + }); } /* ========================= Infrastructure copied from PreparedStatementIT ========================= */ diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/reactive/DefaultReactiveResultSetIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/reactive/DefaultReactiveResultSetIT.java index cfb6a56fac2..c00cf064e51 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/reactive/DefaultReactiveResultSetIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/reactive/DefaultReactiveResultSetIT.java @@ -32,6 +32,7 @@ import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.categories.ParallelizableTests; import com.datastax.oss.driver.internal.core.cql.EmptyColumnDefinitions; @@ -64,20 +65,23 @@ public class DefaultReactiveResultSetIT { @BeforeClass public static void initialize() { CqlSession session = sessionRule.session(); - session.execute("DROP TABLE IF EXISTS test_reactive_read"); - session.execute("DROP TABLE IF EXISTS test_reactive_write"); - session.checkSchemaAgreement(); - session.execute( - SimpleStatement.builder( - "CREATE TABLE test_reactive_read (pk int, cc int, v int, PRIMARY KEY ((pk), cc))") - .setExecutionProfile(sessionRule.slowProfile()) - .build()); - session.execute( - SimpleStatement.builder( - "CREATE TABLE test_reactive_write (pk int, cc int, v int, PRIMARY KEY ((pk), cc))") - .setExecutionProfile(sessionRule.slowProfile()) - .build()); - session.checkSchemaAgreement(); + SchemaChangeSynchronizer.withLock( + () -> { + session.execute("DROP TABLE IF EXISTS test_reactive_read"); + session.execute("DROP TABLE IF EXISTS test_reactive_write"); + session.checkSchemaAgreement(); + session.execute( + SimpleStatement.builder( + "CREATE TABLE test_reactive_read (pk int, cc int, v int, PRIMARY KEY ((pk), cc))") + .setExecutionProfile(sessionRule.slowProfile()) + .build()); + session.execute( + SimpleStatement.builder( + "CREATE TABLE test_reactive_write (pk int, cc int, v int, PRIMARY KEY ((pk), cc))") + .setExecutionProfile(sessionRule.slowProfile()) + .build()); + session.checkSchemaAgreement(); + }); for (int i = 0; i < 1000; i++) { session.execute( SimpleStatement.builder("INSERT INTO test_reactive_read (pk, cc, v) VALUES (0, ?, ?)") diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java index d8239f31872..9fbf5e355eb 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java @@ -28,6 +28,7 @@ import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.ParallelizableTests; @@ -224,15 +225,17 @@ private static String getScriptContents() { private static void setupDatabase() { List statements = STATEMENT_SPLITTER.splitToList(scriptContents); - - // Skip the first statement (CREATE KEYSPACE), we already have a keyspace - for (int i = 1; i < statements.size(); i++) { - String statement = statements.get(i); - try { - SESSION_RULE.session().execute(statement); - } catch (Exception e) { - fail("Error executing statement %s (%s)", statement, e); - } - } + SchemaChangeSynchronizer.withLock( + () -> { + // Skip the first statement (CREATE KEYSPACE), we already have a keyspace + for (int i = 1; i < statements.size(); i++) { + String statement = statements.get(i); + try { + SESSION_RULE.session().execute(statement); + } catch (Exception e) { + fail("Error executing statement %s (%s)", statement, e); + } + } + }); } } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java index 6495b451df7..805b2d970cc 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java @@ -278,6 +278,20 @@ public void should_get_virtual_metadata() { + " total bigint,\n" + " unit text,\n" + " PRIMARY KEY (keyspace_name, table_name, task_id)\n" + + "); */", + // Cassandra 5.0 + "/* VIRTUAL TABLE system_views.sstable_tasks (\n" + + " keyspace_name text,\n" + + " table_name text,\n" + + " task_id timeuuid,\n" + + " completion_ratio double,\n" + + " kind text,\n" + + " progress bigint,\n" + + " sstables int,\n" + + " target_directory text,\n" + + " total bigint,\n" + + " unit text,\n" + + " PRIMARY KEY (keyspace_name, table_name, task_id)\n" + "); */"); // ColumnMetadata is as expected ColumnMetadata cm = tm.getColumn("progress").get(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/type/codec/registry/CodecRegistryIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/type/codec/registry/CodecRegistryIT.java index 2f9a0872b37..74472e8bab9 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/type/codec/registry/CodecRegistryIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/type/codec/registry/CodecRegistryIT.java @@ -38,6 +38,7 @@ import com.datastax.oss.driver.api.core.type.codec.registry.MutableCodecRegistry; import com.datastax.oss.driver.api.core.type.reflect.GenericType; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.ParallelizableTests; @@ -78,35 +79,39 @@ public class CodecRegistryIT { @BeforeClass public static void createSchema() { - // table with simple primary key, single cell. - SESSION_RULE - .session() - .execute( - SimpleStatement.builder("CREATE TABLE IF NOT EXISTS test (k text primary key, v int)") - .setExecutionProfile(SESSION_RULE.slowProfile()) - .build()); - // table with map value - SESSION_RULE - .session() - .execute( - SimpleStatement.builder( - "CREATE TABLE IF NOT EXISTS test2 (k0 text, k1 int, v map, primary key (k0, k1))") - .setExecutionProfile(SESSION_RULE.slowProfile()) - .build()); - // table with UDT - SESSION_RULE - .session() - .execute( - SimpleStatement.builder("CREATE TYPE IF NOT EXISTS coordinates (x int, y int)") - .setExecutionProfile(SESSION_RULE.slowProfile()) - .build()); - SESSION_RULE - .session() - .execute( - SimpleStatement.builder( - "CREATE TABLE IF NOT EXISTS test3 (k0 text, k1 int, v map>, primary key (k0, k1))") - .setExecutionProfile(SESSION_RULE.slowProfile()) - .build()); + SchemaChangeSynchronizer.withLock( + () -> { + // table with simple primary key, single cell. + SESSION_RULE + .session() + .execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS test (k text primary key, v int)") + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + // table with map value + SESSION_RULE + .session() + .execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS test2 (k0 text, k1 int, v map, primary key (k0, k1))") + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + // table with UDT + SESSION_RULE + .session() + .execute( + SimpleStatement.builder("CREATE TYPE IF NOT EXISTS coordinates (x int, y int)") + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + SESSION_RULE + .session() + .execute( + SimpleStatement.builder( + "CREATE TABLE IF NOT EXISTS test3 (k0 text, k1 int, v map>, primary key (k0, k1))") + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + }); } // A simple codec that allows float values to be used for cassandra int column type. diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java index 8918e6020ec..0acdbeae53a 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java @@ -38,11 +38,10 @@ import com.datastax.oss.driver.api.mapper.annotations.Mapper; import com.datastax.oss.driver.api.mapper.annotations.Select; import com.datastax.oss.driver.api.mapper.entity.saving.NullSavingStrategy; -import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; -import com.datastax.oss.driver.categories.ParallelizableTests; import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -51,18 +50,18 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; -@Category(ParallelizableTests.class) +// Do not run LWT tests in parallel because they may interfere. Tests operate on the same row. @BackendRequirement( type = BackendType.CASSANDRA, minInclusive = "3.0", description = ">= in WHERE clause not supported in legacy versions") public class DeleteIT extends InventoryITBase { - private static final CcmRule CCM_RULE = CcmRule.getInstance(); + private static CustomCcmRule CCM_RULE = + CustomCcmRule.builder().withCassandraConfiguration("enable_sasi_indexes", "true").build(); private static final SessionRule SESSION_RULE = SessionRule.builder(CCM_RULE).build(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java index 928fbd6fb8a..3a418c73653 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java @@ -24,6 +24,7 @@ import com.datastax.dse.driver.api.mapper.reactive.MappedReactiveResultSet; import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.Version; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.mapper.annotations.Dao; import com.datastax.oss.driver.api.mapper.annotations.DaoFactory; @@ -34,28 +35,35 @@ import com.datastax.oss.driver.api.mapper.annotations.Mapper; import com.datastax.oss.driver.api.mapper.annotations.Select; import com.datastax.oss.driver.api.mapper.entity.saving.NullSavingStrategy; -import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; +import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; import com.datastax.oss.driver.api.testinfra.session.SessionRule; -import com.datastax.oss.driver.categories.ParallelizableTests; import io.reactivex.Flowable; import java.util.UUID; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; -@Category(ParallelizableTests.class) +// Do not run LWT tests in parallel because they may interfere. Tests operate on the same row. public class DeleteReactiveIT extends InventoryITBase { - private static CcmRule ccmRule = CcmRule.getInstance(); + private static CustomCcmRule ccmRule = configureCcm(CustomCcmRule.builder()).build(); private static SessionRule sessionRule = SessionRule.builder(ccmRule).build(); @ClassRule public static TestRule chain = RuleChain.outerRule(ccmRule).around(sessionRule); + private static CustomCcmRule.Builder configureCcm(CustomCcmRule.Builder builder) { + if (!CcmBridge.DSE_ENABLEMENT + && CcmBridge.VERSION.nextStable().compareTo(Version.V4_0_0) >= 0) { + builder.withCassandraConfiguration("enable_sasi_indexes", true); + } + return builder; + } + private static DseProductDao dao; @BeforeClass diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/EntityPolymorphismIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/EntityPolymorphismIT.java index 08b806af684..3e532e97c00 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/EntityPolymorphismIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/EntityPolymorphismIT.java @@ -47,6 +47,7 @@ import com.datastax.oss.driver.api.mapper.annotations.Update; import com.datastax.oss.driver.api.mapper.entity.saving.NullSavingStrategy; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.categories.ParallelizableTests; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; @@ -83,22 +84,27 @@ public class EntityPolymorphismIT { @BeforeClass public static void setup() { CqlSession session = SESSION_RULE.session(); - for (String query : - ImmutableList.of( - "CREATE TYPE point2d (\"X\" int, \"Y\" int)", - "CREATE TYPE point3d (\"X\" int, \"Y\" int, \"Z\" int)", - "CREATE TABLE circles (circle_id uuid PRIMARY KEY, center2d frozen, radius " - + "double, tags set)", - "CREATE TABLE rectangles (rect_id uuid PRIMARY KEY, bottom_left frozen, top_right frozen, tags set)", - "CREATE TABLE squares (square_id uuid PRIMARY KEY, bottom_left frozen, top_right frozen, tags set)", - "CREATE TABLE spheres (sphere_id uuid PRIMARY KEY, center3d frozen, radius " - + "double, tags set)", - "CREATE TABLE devices (device_id uuid PRIMARY KEY, name text)", - "CREATE TABLE tracked_devices (device_id uuid PRIMARY KEY, name text, location text)", - "CREATE TABLE simple_devices (id uuid PRIMARY KEY, in_use boolean)")) { - session.execute( - SimpleStatement.builder(query).setExecutionProfile(SESSION_RULE.slowProfile()).build()); - } + SchemaChangeSynchronizer.withLock( + () -> { + for (String query : + ImmutableList.of( + "CREATE TYPE point2d (\"X\" int, \"Y\" int)", + "CREATE TYPE point3d (\"X\" int, \"Y\" int, \"Z\" int)", + "CREATE TABLE circles (circle_id uuid PRIMARY KEY, center2d frozen, radius " + + "double, tags set)", + "CREATE TABLE rectangles (rect_id uuid PRIMARY KEY, bottom_left frozen, top_right frozen, tags set)", + "CREATE TABLE squares (square_id uuid PRIMARY KEY, bottom_left frozen, top_right frozen, tags set)", + "CREATE TABLE spheres (sphere_id uuid PRIMARY KEY, center3d frozen, radius " + + "double, tags set)", + "CREATE TABLE devices (device_id uuid PRIMARY KEY, name text)", + "CREATE TABLE tracked_devices (device_id uuid PRIMARY KEY, name text, location text)", + "CREATE TABLE simple_devices (id uuid PRIMARY KEY, in_use boolean)")) { + session.execute( + SimpleStatement.builder(query) + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + } + }); mapper = new EntityPolymorphismIT_TestMapperBuilder(session).build(); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/ImmutableEntityIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/ImmutableEntityIT.java index 555b02c0283..bdfe92a23f9 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/ImmutableEntityIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/ImmutableEntityIT.java @@ -42,6 +42,7 @@ import com.datastax.oss.driver.api.mapper.annotations.Select; import com.datastax.oss.driver.api.mapper.entity.saving.NullSavingStrategy; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.categories.ParallelizableTests; import java.util.Objects; @@ -70,10 +71,15 @@ public class ImmutableEntityIT extends InventoryITBase { public static void setup() { CqlSession session = SESSION_RULE.session(); - for (String query : createStatements(CCM_RULE)) { - session.execute( - SimpleStatement.builder(query).setExecutionProfile(SESSION_RULE.slowProfile()).build()); - } + SchemaChangeSynchronizer.withLock( + () -> { + for (String query : createStatements(CCM_RULE)) { + session.execute( + SimpleStatement.builder(query) + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + } + }); UserDefinedType dimensions2d = session diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java index 75ceee1f2a5..2be025b3739 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java @@ -22,7 +22,7 @@ import com.datastax.oss.driver.api.mapper.annotations.ClusteringColumn; import com.datastax.oss.driver.api.mapper.annotations.Entity; import com.datastax.oss.driver.api.mapper.annotations.PartitionKey; -import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.BaseCcmRule; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import java.util.List; import java.util.Objects; @@ -58,7 +58,7 @@ public abstract class InventoryITBase { protected static ProductSale MP3_DOWNLOAD_SALE_1 = new ProductSale(MP3_DOWNLOAD.getId(), DATE_3, 7, Uuids.startOf(915192000), 0.99, 12); - protected static List createStatements(CcmRule ccmRule) { + protected static List createStatements(BaseCcmRule ccmRule) { ImmutableList.Builder builder = ImmutableList.builder() .add( @@ -92,13 +92,13 @@ protected static List createStatements(CcmRule ccmRule) { private static final Version MINIMUM_SASI_VERSION = Version.parse("3.4.0"); private static final Version BROKEN_SASI_VERSION = Version.parse("6.8.0"); - protected static boolean isSasiBroken(CcmRule ccmRule) { + protected static boolean isSasiBroken(BaseCcmRule ccmRule) { Optional dseVersion = ccmRule.getDseVersion(); // creating SASI indexes is broken in DSE 6.8.0 return dseVersion.isPresent() && dseVersion.get().compareTo(BROKEN_SASI_VERSION) == 0; } - protected static boolean supportsSASI(CcmRule ccmRule) { + protected static boolean supportsSASI(BaseCcmRule ccmRule) { return ccmRule.getCassandraVersion().compareTo(MINIMUM_SASI_VERSION) >= 0; } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/NestedUdtIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/NestedUdtIT.java index 43d41a9c93b..d61b6f6e628 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/NestedUdtIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/NestedUdtIT.java @@ -41,6 +41,7 @@ import com.datastax.oss.driver.api.mapper.annotations.SetEntity; import com.datastax.oss.driver.api.mapper.entity.saving.NullSavingStrategy; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; @@ -119,27 +120,32 @@ public class NestedUdtIT { public static void setup() { CqlSession session = SESSION_RULE.session(); - for (String query : - ImmutableList.of( - "CREATE TYPE type1(s1 text, s2 text)", - "CREATE TYPE type2(i1 int, i2 int)", - "CREATE TYPE type1_partial(s1 text)", - "CREATE TYPE type2_partial(i1 int)", - "CREATE TABLE container(id uuid PRIMARY KEY, " - + "list frozen>, " - + "map1 frozen>>, " - + "map2 frozen>>>," - + "map3 frozen>>>" - + ")", - "CREATE TABLE container_partial(id uuid PRIMARY KEY, " - + "list frozen>, " - + "map1 frozen>>, " - + "map2 frozen>>>," - + "map3 frozen>>>" - + ")")) { - session.execute( - SimpleStatement.builder(query).setExecutionProfile(SESSION_RULE.slowProfile()).build()); - } + SchemaChangeSynchronizer.withLock( + () -> { + for (String query : + ImmutableList.of( + "CREATE TYPE type1(s1 text, s2 text)", + "CREATE TYPE type2(i1 int, i2 int)", + "CREATE TYPE type1_partial(s1 text)", + "CREATE TYPE type2_partial(i1 int)", + "CREATE TABLE container(id uuid PRIMARY KEY, " + + "list frozen>, " + + "map1 frozen>>, " + + "map2 frozen>>>," + + "map3 frozen>>>" + + ")", + "CREATE TABLE container_partial(id uuid PRIMARY KEY, " + + "list frozen>, " + + "map1 frozen>>, " + + "map2 frozen>>>," + + "map3 frozen>>>" + + ")")) { + session.execute( + SimpleStatement.builder(query) + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + } + }); UserDefinedType type1Partial = session diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java index e2a0f1e9987..3df1ccd21a7 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java @@ -35,6 +35,7 @@ import com.datastax.oss.driver.api.mapper.annotations.Mapper; import com.datastax.oss.driver.api.mapper.annotations.Select; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; @@ -72,10 +73,15 @@ public static void setup() { CqlSession session = SESSION_RULE.session(); - for (String query : createStatements(CCM_RULE)) { - session.execute( - SimpleStatement.builder(query).setExecutionProfile(SESSION_RULE.slowProfile()).build()); - } + SchemaChangeSynchronizer.withLock( + () -> { + for (String query : createStatements(CCM_RULE)) { + session.execute( + SimpleStatement.builder(query) + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + } + }); InventoryMapper inventoryMapper = new SelectCustomWhereClauseIT_InventoryMapperBuilder(session).build(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectReactiveIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectReactiveIT.java index 0ea07e552f7..79e4d2b33ea 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectReactiveIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectReactiveIT.java @@ -34,6 +34,7 @@ import com.datastax.oss.driver.api.mapper.annotations.Select; import com.datastax.oss.driver.api.mapper.entity.saving.NullSavingStrategy; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.categories.ParallelizableTests; import io.reactivex.Flowable; @@ -61,10 +62,15 @@ public class SelectReactiveIT extends InventoryITBase { public static void setup() { CqlSession session = sessionRule.session(); - for (String query : createStatements(ccmRule)) { - session.execute( - SimpleStatement.builder(query).setExecutionProfile(sessionRule.slowProfile()).build()); - } + SchemaChangeSynchronizer.withLock( + () -> { + for (String query : createStatements(ccmRule)) { + session.execute( + SimpleStatement.builder(query) + .setExecutionProfile(sessionRule.slowProfile()) + .build()); + } + }); DseInventoryMapper inventoryMapper = new SelectReactiveIT_DseInventoryMapperBuilder(session).build(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/UpdateIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/UpdateIT.java index 27b4d6e9d90..3fac733c900 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/UpdateIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/UpdateIT.java @@ -37,6 +37,7 @@ import com.datastax.oss.driver.api.mapper.annotations.Update; import com.datastax.oss.driver.api.mapper.entity.saving.NullSavingStrategy; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.categories.ParallelizableTests; import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; @@ -65,14 +66,18 @@ public class UpdateIT extends InventoryITBase { @BeforeClass public static void setup() { CqlSession session = SESSION_RULE.session(); - - for (String query : createStatements(CCM_RULE)) { - session.execute( - SimpleStatement.builder(query).setExecutionProfile(SESSION_RULE.slowProfile()).build()); - } - session.execute( - SimpleStatement.newInstance("CREATE TABLE only_p_k(id uuid PRIMARY KEY)") - .setExecutionProfile(SESSION_RULE.slowProfile())); + SchemaChangeSynchronizer.withLock( + () -> { + for (String query : createStatements(CCM_RULE)) { + session.execute( + SimpleStatement.builder(query) + .setExecutionProfile(SESSION_RULE.slowProfile()) + .build()); + } + session.execute( + SimpleStatement.newInstance("CREATE TABLE only_p_k(id uuid PRIMARY KEY)") + .setExecutionProfile(SESSION_RULE.slowProfile())); + }); inventoryMapper = new UpdateIT_InventoryMapperBuilder(session).build(); dao = inventoryMapper.productDao(SESSION_RULE.keyspace()); diff --git a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java index 8a520488e5c..8b140930870 100644 --- a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java +++ b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java @@ -81,7 +81,7 @@ public synchronized void afterSuite() { if (running) { LOGGER.info("Stopping CCM"); CCM_BRIDGE.stop(); - CCM_BRIDGE.remove(); + CCM_BRIDGE.close(); running = false; LOGGER.info("CCM stopped"); } diff --git a/pom.xml b/pom.xml index 082daeb3566..94311719e5f 100644 --- a/pom.xml +++ b/pom.xml @@ -991,6 +991,17 @@ limitations under the License.]]> [11,) + + + test-jdk-14 + + [14,) + + + + -XX:+AllowRedefinitionToAddDeleteMethods + + test-jdk-17 diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java index b8b684ee5b2..65210acd2a2 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java @@ -38,7 +38,7 @@ public abstract class BaseCcmRule extends CassandraResourceRule { new Thread( () -> { try { - ccmBridge.remove(); + ccmBridge.close(); } catch (Exception e) { // silently remove as may have already been removed. } @@ -53,7 +53,7 @@ protected void before() { @Override protected void after() { - ccmBridge.remove(); + ccmBridge.close(); } @Override diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java index 98739e7715d..995513e3919 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java @@ -197,9 +197,10 @@ public Version getCassandraVersion() { } private String getCcmVersionString(Version version) { - // for 4.0 pre-releases, the CCM version string needs to be "4.0-alpha1" or "4.0-alpha2" - // Version.toString() always adds a patch value, even if it's not specified when parsing. - if (version.getMajor() == 4 + // for 4.0 or 5.0 pre-releases, the CCM version string needs to be "4.0-alpha1", "4.0-alpha2" or + // "5.0-beta1" Version.toString() always adds a patch value, even if it's not specified when + // parsing. + if (version.getMajor() >= 4 && version.getMinor() == 0 && version.getPatch() == 0 && version.getPreReleaseLabels() != null) { @@ -292,8 +293,7 @@ public void reloadCore(int node, String keyspace, String table, boolean reindex) public void start() { if (started.compareAndSet(false, true)) { List cmdAndArgs = Lists.newArrayList("start", jvmArgs, "--wait-for-binary-proto"); - overrideJvmVersionForDseWorkloads() - .ifPresent(jvmVersion -> cmdAndArgs.add(String.format("--jvm_version=%d", jvmVersion))); + updateJvmVersion(cmdAndArgs); try { execute(cmdAndArgs.toArray(new String[0])); } catch (RuntimeException re) { @@ -324,9 +324,13 @@ public void resume(int n) { public void start(int n) { List cmdAndArgs = Lists.newArrayList("node" + n, "start"); + updateJvmVersion(cmdAndArgs); + execute(cmdAndArgs.toArray(new String[0])); + } + + private void updateJvmVersion(List cmdAndArgs) { overrideJvmVersionForDseWorkloads() .ifPresent(jvmVersion -> cmdAndArgs.add(String.format("--jvm_version=%d", jvmVersion))); - execute(cmdAndArgs.toArray(new String[0])); } public void stop(int n) { @@ -423,7 +427,9 @@ protected void processLine(String line, int logLevel) { @Override public void close() { - remove(); + if (created.compareAndSet(true, false)) { + remove(); + } } /** diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/SchemaChangeSynchronizer.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/SchemaChangeSynchronizer.java new file mode 100644 index 00000000000..093d1d3f9f9 --- /dev/null +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/SchemaChangeSynchronizer.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.testinfra.ccm; + +import java.util.concurrent.Semaphore; + +/** + * Running multiple parallel integration tests may fail due to query timeout when trying to apply + * several schema changes at once. Limit concurrently executed DDLs to 5. + */ +public class SchemaChangeSynchronizer { + private static final Semaphore lock = new Semaphore(5); + + public static void withLock(Runnable callback) { + try { + lock.acquire(); + try { + callback.run(); + } finally { + lock.release(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Thread interrupted wile waiting to obtain DDL lock", e); + } + } +} diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java index ce3903bcfcb..5396e5c6cc6 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.api.testinfra.CassandraResourceRule; import com.datastax.oss.driver.api.testinfra.ccm.BaseCcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; import com.datastax.oss.driver.api.testinfra.simulacron.SimulacronRule; import java.util.Objects; import java.util.Optional; @@ -195,7 +196,10 @@ protected void after() { ScriptGraphStatement.SYNC); } if (keyspace != null) { - SessionUtils.dropKeyspace(session, keyspace, slowProfile); + SchemaChangeSynchronizer.withLock( + () -> { + SessionUtils.dropKeyspace(session, keyspace, slowProfile); + }); } session.close(); } From 811acb2fe77464f679a09226a03c1995694c51b4 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Tue, 4 Jun 2024 10:45:28 +0200 Subject: [PATCH 02/48] CASSANDRA-19635: Configure Jenkins to run integration tests with C* 5.x patch by Lukasz Antoniak; reviewed by Bret McGuire for CASSANDRA-19635 --- Jenkinsfile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d38b7c63849..4f1ef95d101 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -261,7 +261,7 @@ pipeline { '3.11', // Previous Apache CassandraⓇ '4.0', // Previous Apache CassandraⓇ '4.1', // Current Apache CassandraⓇ - '5.0', // Development Apache CassandraⓇ + '5.0-beta1', // Development Apache CassandraⓇ 'dse-4.8.16', // Previous EOSL DataStax Enterprise 'dse-5.0.15', // Long Term Support DataStax Enterprise 'dse-5.1.35', // Legacy DataStax Enterprise @@ -411,14 +411,14 @@ pipeline { triggers { // schedules only run against release branches (i.e. 3.x, 4.x, 4.5.x, etc.) parameterizedCron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? """ - # Every weeknight (Monday - Friday) around 2:00 AM - ### JDK8 tests against 2.1, 3.0, DSE 4.8, DSE 5.0, DSE 5.1, dse-6.0.18 and DSE 6.7 - H 2 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 - ### JDK11 tests against 3.11, 4.0 and DSE 6.8 - H 2 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.0 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 - # Every weekend (Sunday) around 12:00 PM noon - ### JDK14 tests against 3.11, 4.0 and DSE 6.8 - H 12 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.0 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.14 + # Every weekend (Saturday, Sunday) around 2:00 AM + ### JDK8 tests against 2.1, 3.0, 4.0, DSE 4.8, DSE 5.0, DSE 5.1, dse-6.0.18 and DSE 6.7 + H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 4.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 + # Every weeknight (Monday - Friday) around 12:00 PM noon + ### JDK11 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 + ### JDK17 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 """ : "") } @@ -452,8 +452,8 @@ pipeline { axes { axis { name 'SERVER_VERSION' - values '3.11', // Latest stable Apache CassandraⓇ - '4.1', // Development Apache CassandraⓇ + values '3.11', // Latest stable Apache CassandraⓇ + '4.1', // Development Apache CassandraⓇ 'dse-6.8.30' // Current DataStax Enterprise } axis { @@ -565,7 +565,7 @@ pipeline { '3.11', // Previous Apache CassandraⓇ '4.0', // Previous Apache CassandraⓇ '4.1', // Current Apache CassandraⓇ - '5.0', // Development Apache CassandraⓇ + '5.0-beta1', // Development Apache CassandraⓇ 'dse-4.8.16', // Previous EOSL DataStax Enterprise 'dse-5.0.15', // Last EOSL DataStax Enterprise 'dse-5.1.35', // Legacy DataStax Enterprise From 85bb4065098b887d2dda26eb14423ce4fc687045 Mon Sep 17 00:00:00 2001 From: Brad Schoening Date: Tue, 4 Jun 2024 17:30:41 -0400 Subject: [PATCH 03/48] update badge URL to org.apache.cassandra/java-driver-core --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c53c8f2db29..2a30cb68c9a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ :warning: The java-driver has recently been donated by Datastax to The Apache Software Foundation and the Apache Cassandra project. Bear with us as we move assets and coordinates. -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.datastax.oss/java-driver-core/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.datastax.oss/java-driver-core) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.cassandra/java-driver-core/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.apache.cassandra/java-driver-core) *If you're reading this on github.com, please note that this is the readme for the development version and that some features described here might not yet have been released. You can find the From d0a1e44a4415c7a0489f8c35ee9ce49e20d7bc61 Mon Sep 17 00:00:00 2001 From: Benoit Tellier Date: Sun, 22 Jan 2023 13:58:19 +0700 Subject: [PATCH 04/48] Limit calls to Conversions.resolveExecutionProfile Those repeated calls account for a non-negligible portion of my application CPU (0.6%) and can definitly be a final field so that it gets resolved only once per CqlRequestHandler. patch by Benoit Tellier; reviewed by Andy Tolbert, and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1623 --- .../ContinuousRequestHandlerBase.java | 15 +++--- .../core/graph/GraphRequestHandler.java | 15 +++--- .../driver/internal/core/cql/Conversions.java | 38 +++++++++++-- .../internal/core/cql/CqlPrepareHandler.java | 11 ++-- .../internal/core/cql/CqlRequestHandler.java | 54 +++++++------------ 5 files changed, 75 insertions(+), 58 deletions(-) diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java b/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java index 44df3b3a03d..9a7be344721 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java @@ -648,12 +648,13 @@ public void operationComplete(@NonNull Future future) { } } else { LOG.trace("[{}] Request sent on {}", logPrefix, channel); - if (scheduleSpeculativeExecution && Conversions.resolveIdempotence(statement, context)) { + if (scheduleSpeculativeExecution + && Conversions.resolveIdempotence(statement, executionProfile)) { int nextExecution = executionIndex + 1; // Note that `node` is the first node of the execution, it might not be the "slow" one // if there were retries, but in practice retries are rare. long nextDelay = - Conversions.resolveSpeculativeExecutionPolicy(statement, context) + Conversions.resolveSpeculativeExecutionPolicy(context, executionProfile) .nextExecution(node, keyspace, statement, nextExecution); if (nextDelay >= 0) { scheduleSpeculativeExecution(nextExecution, nextDelay); @@ -787,12 +788,12 @@ public void onFailure(@NonNull Throwable error) { cancelTimeout(pageTimeout); LOG.trace(String.format("[%s] Request failure", logPrefix), error); RetryVerdict verdict; - if (!Conversions.resolveIdempotence(statement, context) + if (!Conversions.resolveIdempotence(statement, executionProfile) || error instanceof FrameTooLongException) { verdict = RetryVerdict.RETHROW; } else { try { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); verdict = retryPolicy.onRequestAbortedVerdict(statement, error, retryCount); } catch (Throwable cause) { abort( @@ -945,7 +946,7 @@ private void processRecoverableError(@NonNull CoordinatorException error) { assert lock.isHeldByCurrentThread(); NodeMetricUpdater metricUpdater = ((DefaultNode) node).getMetricUpdater(); RetryVerdict verdict; - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); if (error instanceof ReadTimeoutException) { ReadTimeoutException readTimeout = (ReadTimeoutException) error; verdict = @@ -964,7 +965,7 @@ private void processRecoverableError(@NonNull CoordinatorException error) { DefaultNodeMetric.IGNORES_ON_READ_TIMEOUT); } else if (error instanceof WriteTimeoutException) { WriteTimeoutException writeTimeout = (WriteTimeoutException) error; - if (Conversions.resolveIdempotence(statement, context)) { + if (Conversions.resolveIdempotence(statement, executionProfile)) { verdict = retryPolicy.onWriteTimeoutVerdict( statement, @@ -999,7 +1000,7 @@ private void processRecoverableError(@NonNull CoordinatorException error) { DefaultNodeMetric.IGNORES_ON_UNAVAILABLE); } else { verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onErrorResponseVerdict(statement, error, retryCount) : RetryVerdict.RETHROW; updateErrorMetrics( diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java b/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java index c2298458805..702da69b855 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java @@ -557,12 +557,13 @@ public void operationComplete(Future future) { cancel(); } else { inFlightCallbacks.add(this); - if (scheduleNextExecution && Conversions.resolveIdempotence(statement, context)) { + if (scheduleNextExecution + && Conversions.resolveIdempotence(statement, executionProfile)) { int nextExecution = execution + 1; long nextDelay; try { nextDelay = - Conversions.resolveSpeculativeExecutionPolicy(statement, context) + Conversions.resolveSpeculativeExecutionPolicy(context, executionProfile) .nextExecution(node, null, statement, nextExecution); } catch (Throwable cause) { // This is a bug in the policy, but not fatal since we have at least one other @@ -678,7 +679,7 @@ private void processErrorResponse(Error errorMessage) { trackNodeError(node, error, NANOTIME_NOT_MEASURED_YET); setFinalError(statement, error, node, execution); } else { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); RetryVerdict verdict; if (error instanceof ReadTimeoutException) { ReadTimeoutException readTimeout = (ReadTimeoutException) error; @@ -699,7 +700,7 @@ private void processErrorResponse(Error errorMessage) { } else if (error instanceof WriteTimeoutException) { WriteTimeoutException writeTimeout = (WriteTimeoutException) error; verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onWriteTimeoutVerdict( statement, writeTimeout.getConsistencyLevel(), @@ -731,7 +732,7 @@ private void processErrorResponse(Error errorMessage) { DefaultNodeMetric.IGNORES_ON_UNAVAILABLE); } else { verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onErrorResponseVerdict(statement, error, retryCount) : RetryVerdict.RETHROW; updateErrorMetrics( @@ -810,12 +811,12 @@ public void onFailure(Throwable error) { } LOG.trace("[{}] Request failure, processing: {}", logPrefix, error); RetryVerdict verdict; - if (!Conversions.resolveIdempotence(statement, context) + if (!Conversions.resolveIdempotence(statement, executionProfile) || error instanceof FrameTooLongException) { verdict = RetryVerdict.RETHROW; } else { try { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); verdict = retryPolicy.onRequestAbortedVerdict(statement, error, retryCount); } catch (Throwable cause) { setFinalError( diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java index 529664c6666..ff9384b3e24 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java @@ -535,29 +535,59 @@ public static CoordinatorException toThrowable( } } + /** Use {@link #resolveIdempotence(Request, DriverExecutionProfile)} instead. */ + @Deprecated public static boolean resolveIdempotence(Request request, InternalDriverContext context) { + return resolveIdempotence(request, resolveExecutionProfile(request, context)); + } + + public static boolean resolveIdempotence( + Request request, DriverExecutionProfile executionProfile) { Boolean requestIsIdempotent = request.isIdempotent(); - DriverExecutionProfile executionProfile = resolveExecutionProfile(request, context); return (requestIsIdempotent == null) ? executionProfile.getBoolean(DefaultDriverOption.REQUEST_DEFAULT_IDEMPOTENCE) : requestIsIdempotent; } + /** Use {@link #resolveRequestTimeout(Request, DriverExecutionProfile)} instead. */ + @Deprecated public static Duration resolveRequestTimeout(Request request, InternalDriverContext context) { - DriverExecutionProfile executionProfile = resolveExecutionProfile(request, context); - return request.getTimeout() != null - ? request.getTimeout() + return resolveRequestTimeout(request, resolveExecutionProfile(request, context)); + } + + public static Duration resolveRequestTimeout( + Request request, DriverExecutionProfile executionProfile) { + Duration timeout = request.getTimeout(); + return timeout != null + ? timeout : executionProfile.getDuration(DefaultDriverOption.REQUEST_TIMEOUT); } + /** Use {@link #resolveRetryPolicy(InternalDriverContext, DriverExecutionProfile)} instead. */ + @Deprecated public static RetryPolicy resolveRetryPolicy(Request request, InternalDriverContext context) { DriverExecutionProfile executionProfile = resolveExecutionProfile(request, context); return context.getRetryPolicy(executionProfile.getName()); } + public static RetryPolicy resolveRetryPolicy( + InternalDriverContext context, DriverExecutionProfile executionProfile) { + return context.getRetryPolicy(executionProfile.getName()); + } + + /** + * Use {@link #resolveSpeculativeExecutionPolicy(InternalDriverContext, DriverExecutionProfile)} + * instead. + */ + @Deprecated public static SpeculativeExecutionPolicy resolveSpeculativeExecutionPolicy( Request request, InternalDriverContext context) { DriverExecutionProfile executionProfile = resolveExecutionProfile(request, context); return context.getSpeculativeExecutionPolicy(executionProfile.getName()); } + + public static SpeculativeExecutionPolicy resolveSpeculativeExecutionPolicy( + InternalDriverContext context, DriverExecutionProfile executionProfile) { + return context.getSpeculativeExecutionPolicy(executionProfile.getName()); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java index 6faa8eee59f..8fe1adb20b1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java @@ -92,6 +92,7 @@ public class CqlPrepareHandler implements Throttled { private final Timeout scheduledTimeout; private final RequestThrottler throttler; private final Boolean prepareOnAllNodes; + private final DriverExecutionProfile executionProfile; private volatile InitialPrepareCallback initialCallback; // The errors on the nodes that were already tried (lazily initialized on the first error). @@ -111,7 +112,7 @@ protected CqlPrepareHandler( this.initialRequest = request; this.session = session; this.context = context; - DriverExecutionProfile executionProfile = Conversions.resolveExecutionProfile(request, context); + executionProfile = Conversions.resolveExecutionProfile(request, context); this.queryPlan = context .getLoadBalancingPolicyWrapper() @@ -131,7 +132,7 @@ protected CqlPrepareHandler( }); this.timer = context.getNettyOptions().getTimer(); - Duration timeout = Conversions.resolveRequestTimeout(request, context); + Duration timeout = Conversions.resolveRequestTimeout(request, executionProfile); this.scheduledTimeout = scheduleTimeout(timeout); this.prepareOnAllNodes = executionProfile.getBoolean(DefaultDriverOption.PREPARE_ON_ALL_NODES); @@ -292,7 +293,7 @@ private CompletionStage prepareOnOtherNode(PrepareRequest request, Node no false, toPrepareMessage(request), request.getCustomPayload(), - Conversions.resolveRequestTimeout(request, context), + Conversions.resolveRequestTimeout(request, executionProfile), throttler, session.getMetricUpdater(), logPrefix); @@ -419,7 +420,7 @@ private void processErrorResponse(Error errorMessage) { } else { // Because prepare requests are known to always be idempotent, we call the retry policy // directly, without checking the flag. - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(request, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); RetryVerdict verdict = retryPolicy.onErrorResponseVerdict(request, error, retryCount); processRetryVerdict(verdict, error); } @@ -457,7 +458,7 @@ public void onFailure(Throwable error) { LOG.trace("[{}] Request failure, processing: {}", logPrefix, error.toString()); RetryVerdict verdict; try { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(request, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); verdict = retryPolicy.onRequestAbortedVerdict(request, error, retryCount); } catch (Throwable cause) { setFinalError( diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java index e7e334d57d8..a1c6b0e5466 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java @@ -126,6 +126,7 @@ public class CqlRequestHandler implements Throttled { private final RequestThrottler throttler; private final RequestTracker requestTracker; private final SessionMetricUpdater sessionMetricUpdater; + private final DriverExecutionProfile executionProfile; // The errors on the nodes that were already tried (lazily initialized on the first error). // We don't use a map because nodes can appear multiple times. @@ -167,7 +168,8 @@ protected CqlRequestHandler( this.sessionMetricUpdater = session.getMetricUpdater(); this.timer = context.getNettyOptions().getTimer(); - Duration timeout = Conversions.resolveRequestTimeout(statement, context); + this.executionProfile = Conversions.resolveExecutionProfile(initialStatement, context); + Duration timeout = Conversions.resolveRequestTimeout(statement, executionProfile); this.scheduledTimeout = scheduleTimeout(timeout); this.throttler = context.getRequestThrottler(); @@ -176,8 +178,6 @@ protected CqlRequestHandler( @Override public void onThrottleReady(boolean wasDelayed) { - DriverExecutionProfile executionProfile = - Conversions.resolveExecutionProfile(initialStatement, context); if (wasDelayed // avoid call to nanoTime() if metric is disabled: && sessionMetricUpdater.isEnabled( @@ -276,8 +276,6 @@ private void sendRequest( retryCount, scheduleNextExecution, logPrefix); - DriverExecutionProfile executionProfile = - Conversions.resolveExecutionProfile(statement, context); Message message = Conversions.toMessage(statement, executionProfile, context); channel .write(message, statement.isTracing(), statement.getCustomPayload(), nodeResponseCallback) @@ -336,37 +334,28 @@ private void setFinalResult( totalLatencyNanos = completionTimeNanos - startTimeNanos; long nodeLatencyNanos = completionTimeNanos - callback.nodeStartTimeNanos; requestTracker.onNodeSuccess( - callback.statement, - nodeLatencyNanos, - callback.executionProfile, - callback.node, - logPrefix); + callback.statement, nodeLatencyNanos, executionProfile, callback.node, logPrefix); requestTracker.onSuccess( - callback.statement, - totalLatencyNanos, - callback.executionProfile, - callback.node, - logPrefix); + callback.statement, totalLatencyNanos, executionProfile, callback.node, logPrefix); } if (sessionMetricUpdater.isEnabled( - DefaultSessionMetric.CQL_REQUESTS, callback.executionProfile.getName())) { + DefaultSessionMetric.CQL_REQUESTS, executionProfile.getName())) { if (completionTimeNanos == NANOTIME_NOT_MEASURED_YET) { completionTimeNanos = System.nanoTime(); totalLatencyNanos = completionTimeNanos - startTimeNanos; } sessionMetricUpdater.updateTimer( DefaultSessionMetric.CQL_REQUESTS, - callback.executionProfile.getName(), + executionProfile.getName(), totalLatencyNanos, TimeUnit.NANOSECONDS); } } // log the warnings if they have NOT been disabled if (!executionInfo.getWarnings().isEmpty() - && callback.executionProfile.getBoolean(DefaultDriverOption.REQUEST_LOG_WARNINGS) + && executionProfile.getBoolean(DefaultDriverOption.REQUEST_LOG_WARNINGS) && LOG.isWarnEnabled()) { - logServerWarnings( - callback.statement, callback.executionProfile, executionInfo.getWarnings()); + logServerWarnings(callback.statement, executionProfile, executionInfo.getWarnings()); } } catch (Throwable error) { setFinalError(callback.statement, error, callback.node, -1); @@ -418,21 +407,17 @@ private ExecutionInfo buildExecutionInfo( schemaInAgreement, session, context, - callback.executionProfile); + executionProfile); } @Override public void onThrottleFailure(@NonNull RequestThrottlingException error) { - DriverExecutionProfile executionProfile = - Conversions.resolveExecutionProfile(initialStatement, context); sessionMetricUpdater.incrementCounter( DefaultSessionMetric.THROTTLING_ERRORS, executionProfile.getName()); setFinalError(initialStatement, error, null, -1); } private void setFinalError(Statement statement, Throwable error, Node node, int execution) { - DriverExecutionProfile executionProfile = - Conversions.resolveExecutionProfile(statement, context); if (error instanceof DriverException) { ((DriverException) error) .setExecutionInfo( @@ -475,7 +460,6 @@ private class NodeResponseCallback private final long nodeStartTimeNanos = System.nanoTime(); private final Statement statement; - private final DriverExecutionProfile executionProfile; private final Node node; private final Queue queryPlan; private final DriverChannel channel; @@ -505,7 +489,6 @@ private NodeResponseCallback( this.retryCount = retryCount; this.scheduleNextExecution = scheduleNextExecution; this.logPrefix = logPrefix + "|" + execution; - this.executionProfile = Conversions.resolveExecutionProfile(statement, context); } // this gets invoked once the write completes. @@ -544,12 +527,13 @@ public void operationComplete(Future future) throws Exception { cancel(); } else { inFlightCallbacks.add(this); - if (scheduleNextExecution && Conversions.resolveIdempotence(statement, context)) { + if (scheduleNextExecution + && Conversions.resolveIdempotence(statement, executionProfile)) { int nextExecution = execution + 1; long nextDelay; try { nextDelay = - Conversions.resolveSpeculativeExecutionPolicy(statement, context) + Conversions.resolveSpeculativeExecutionPolicy(context, executionProfile) .nextExecution(node, keyspace, statement, nextExecution); } catch (Throwable cause) { // This is a bug in the policy, but not fatal since we have at least one other @@ -697,7 +681,7 @@ private void processErrorResponse(Error errorMessage) { true, reprepareMessage, repreparePayload.customPayload, - Conversions.resolveRequestTimeout(statement, context), + Conversions.resolveRequestTimeout(statement, executionProfile), throttler, sessionMetricUpdater, logPrefix); @@ -767,7 +751,7 @@ private void processErrorResponse(Error errorMessage) { trackNodeError(node, error, NANOTIME_NOT_MEASURED_YET); setFinalError(statement, error, node, execution); } else { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); RetryVerdict verdict; if (error instanceof ReadTimeoutException) { ReadTimeoutException readTimeout = (ReadTimeoutException) error; @@ -788,7 +772,7 @@ private void processErrorResponse(Error errorMessage) { } else if (error instanceof WriteTimeoutException) { WriteTimeoutException writeTimeout = (WriteTimeoutException) error; verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onWriteTimeoutVerdict( statement, writeTimeout.getConsistencyLevel(), @@ -820,7 +804,7 @@ private void processErrorResponse(Error errorMessage) { DefaultNodeMetric.IGNORES_ON_UNAVAILABLE); } else { verdict = - Conversions.resolveIdempotence(statement, context) + Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onErrorResponseVerdict(statement, error, retryCount) : RetryVerdict.RETHROW; updateErrorMetrics( @@ -899,12 +883,12 @@ public void onFailure(Throwable error) { } LOG.trace("[{}] Request failure, processing: {}", logPrefix, error); RetryVerdict verdict; - if (!Conversions.resolveIdempotence(statement, context) + if (!Conversions.resolveIdempotence(statement, executionProfile) || error instanceof FrameTooLongException) { verdict = RetryVerdict.RETHROW; } else { try { - RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(statement, context); + RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); verdict = retryPolicy.onRequestAbortedVerdict(statement, error, retryCount); } catch (Throwable cause) { setFinalError( From a17f7be614a09ab81bc2982b7f7ab3a123b4ab28 Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Thu, 22 Aug 2024 14:28:46 +0200 Subject: [PATCH 05/48] autolink JIRA tickets in commit messages patch by Stefan Miklosovic; reviewed by Michael Semb Wever for CASSANDRA-19854 --- .asf.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 5ebca4b6e33..ad58f536398 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -19,6 +19,7 @@ notifications: commits: commits@cassandra.apache.org issues: commits@cassandra.apache.org pullrequests: pr@cassandra.apache.org + jira_options: link worklog github: description: "Java Driver for Apache Cassandra®" @@ -31,6 +32,5 @@ github: wiki: false issues: false projects: false - -notifications: - jira_options: link worklog + autolink_jira: + - CASSANDRA From 0962794b2ec724d9939cd47380e68b979b46f693 Mon Sep 17 00:00:00 2001 From: Ammar Khaku Date: Sun, 20 Nov 2022 19:14:07 -0800 Subject: [PATCH 06/48] Don't return empty routing key when partition key is unbound DefaultBoundStatement#getRoutingKey has logic to infer the routing key when no one has explicitly called setRoutingKey or otherwise set the routing key on the statement. It however doesn't check for cases where nothing has been bound yet on the statement. This causes more problems if the user decides to get a BoundStatementBuilder from the PreparedStatement, set some fields on it, and then copy it by constructing new BoundStatementBuilder objects with the BoundStatement as a parameter, since the empty ByteBuffer gets copied to all bound statements, resulting in all requests being targeted to the same Cassandra node in a token-aware load balancing policy. patch by Ammar Khaku; reviewed by Andy Tolbert, and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1620 --- .../core/cql/DefaultBoundStatement.java | 3 ++- .../driver/core/cql/PreparedStatementIT.java | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/DefaultBoundStatement.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/DefaultBoundStatement.java index fb6b8fd7b27..3cf99c1be6e 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/DefaultBoundStatement.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/DefaultBoundStatement.java @@ -360,7 +360,8 @@ public ByteBuffer getRoutingKey() { if (indices.isEmpty()) { return null; } else if (indices.size() == 1) { - return getBytesUnsafe(indices.get(0)); + int index = indices.get(0); + return isSet(index) ? getBytesUnsafe(index) : null; } else { ByteBuffer[] components = new ByteBuffer[indices.size()]; for (int i = 0; i < components.length; i++) { diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementIT.java index c0df01e3519..5671a7684e5 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementIT.java @@ -527,6 +527,25 @@ private void should_infer_routing_information_when_partition_key_is_bound(String assertThat(tokenFactory.hash(boundStatement.getRoutingKey())).isEqualTo(expectedToken); } + @Test + public void should_return_null_routing_information_when_single_partition_key_is_unbound() { + should_return_null_routing_information_when_single_partition_key_is_unbound( + "SELECT a FROM prepared_statement_test WHERE a = ?"); + should_return_null_routing_information_when_single_partition_key_is_unbound( + "INSERT INTO prepared_statement_test (a) VALUES (?)"); + should_return_null_routing_information_when_single_partition_key_is_unbound( + "UPDATE prepared_statement_test SET b = 1 WHERE a = ?"); + should_return_null_routing_information_when_single_partition_key_is_unbound( + "DELETE FROM prepared_statement_test WHERE a = ?"); + } + + private void should_return_null_routing_information_when_single_partition_key_is_unbound( + String queryString) { + CqlSession session = sessionRule.session(); + BoundStatement boundStatement = session.prepare(queryString).bind(); + assertThat(boundStatement.getRoutingKey()).isNull(); + } + private static Iterable firstPageOf(CompletionStage stage) { return CompletableFutures.getUninterruptibly(stage).currentPage(); } From e6ae1933667066bf16ac3ac5203a2a6fdadd1946 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Fri, 16 Aug 2024 11:06:09 +0200 Subject: [PATCH 07/48] JAVA-3167: CompletableFutures.allSuccessful() may return never completed future patch by Lukasz Antoniak; reviewed by Andy Tolbert, and Bret McGuire for JAVA-3167 --- .../util/concurrent/CompletableFutures.java | 5 +++- .../concurrent/CompletableFuturesTest.java | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFutures.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFutures.java index 03265bd1d77..275b2ddfeef 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFutures.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFutures.java @@ -100,7 +100,10 @@ public static CompletionStage allSuccessful(List> i } else { Throwable finalError = errors.get(0); for (int i = 1; i < errors.size(); i++) { - finalError.addSuppressed(errors.get(i)); + Throwable suppressedError = errors.get(i); + if (finalError != suppressedError) { + finalError.addSuppressed(suppressedError); + } } result.completeExceptionally(finalError); } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java new file mode 100644 index 00000000000..8a710e02d50 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java @@ -0,0 +1,29 @@ +package com.datastax.oss.driver.internal.core.util.concurrent; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +public class CompletableFuturesTest { + @Test + public void should_not_suppress_identical_exceptions() throws Exception { + RuntimeException error = new RuntimeException(); + CompletableFuture future1 = new CompletableFuture<>(); + future1.completeExceptionally(error); + CompletableFuture future2 = new CompletableFuture<>(); + future2.completeExceptionally(error); + try { + // if timeout exception is thrown, it indicates that CompletableFutures.allSuccessful() + // did not complete the returned future and potentially caller will wait infinitely + CompletableFutures.allSuccessful(Arrays.asList(future1, future2)) + .toCompletableFuture() + .get(1, TimeUnit.SECONDS); + } catch (ExecutionException e) { + assertThat(e.getCause()).isEqualTo(error); + } + } +} From 5ee12acfff720047db7611a6f54450c1646031a3 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Tue, 3 Sep 2024 16:06:04 -0500 Subject: [PATCH 08/48] ninja-fix Various test fixes --- .../concurrent/CompletableFuturesTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java index 8a710e02d50..04f96f185fd 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/util/concurrent/CompletableFuturesTest.java @@ -1,6 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.datastax.oss.driver.internal.core.util.concurrent; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; import java.util.Arrays; import java.util.concurrent.CompletableFuture; @@ -22,6 +40,7 @@ public void should_not_suppress_identical_exceptions() throws Exception { CompletableFutures.allSuccessful(Arrays.asList(future1, future2)) .toCompletableFuture() .get(1, TimeUnit.SECONDS); + fail(); } catch (ExecutionException e) { assertThat(e.getCause()).isEqualTo(error); } From 9cfb4f6712e392c1b6c87db268565fd3b27d0c5c Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Thu, 5 Sep 2024 13:04:52 +0200 Subject: [PATCH 09/48] Run integration tests with DSE 6.9.0 patch by Lukasz Antoniak; reviewed by Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1955 --- Jenkinsfile | 19 +++++++++++++------ .../datastax/oss/driver/api/core/Version.java | 1 + .../driver/api/testinfra/ccm/CcmBridge.java | 3 ++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4f1ef95d101..4cc20d79604 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -268,6 +268,7 @@ pipeline { 'dse-6.0.18', // Previous DataStax Enterprise 'dse-6.7.17', // Previous DataStax Enterprise 'dse-6.8.30', // Current DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise 'ALL'], description: '''Apache Cassandra® and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS builds @@ -325,6 +326,10 @@ pipeline { + + + +
dse-6.8.30 DataStax Enterprise v6.8.x
dse-6.9.0DataStax Enterprise v6.9.x
''') choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_JABBA_VERSION', @@ -416,9 +421,9 @@ pipeline { H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 4.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 # Every weeknight (Monday - Friday) around 12:00 PM noon ### JDK11 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 ### JDK17 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 """ : "") } @@ -452,9 +457,10 @@ pipeline { axes { axis { name 'SERVER_VERSION' - values '3.11', // Latest stable Apache CassandraⓇ - '4.1', // Development Apache CassandraⓇ - 'dse-6.8.30' // Current DataStax Enterprise + values '3.11', // Latest stable Apache CassandraⓇ + '4.1', // Development Apache CassandraⓇ + 'dse-6.8.30', // Current DataStax Enterprise + 'dse-6.9.0' // Current DataStax Enterprise } axis { name 'JABBA_VERSION' @@ -571,7 +577,8 @@ pipeline { 'dse-5.1.35', // Legacy DataStax Enterprise 'dse-6.0.18', // Previous DataStax Enterprise 'dse-6.7.17', // Previous DataStax Enterprise - 'dse-6.8.30' // Current DataStax Enterprise + 'dse-6.8.30', // Current DataStax Enterprise + 'dse-6.9.0' // Current DataStax Enterprise } } when { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/Version.java b/core/src/main/java/com/datastax/oss/driver/api/core/Version.java index 3f12c54faf7..4de006da268 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/Version.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/Version.java @@ -56,6 +56,7 @@ public class Version implements Comparable, Serializable { @NonNull public static final Version V5_0_0 = Objects.requireNonNull(parse("5.0.0")); @NonNull public static final Version V6_7_0 = Objects.requireNonNull(parse("6.7.0")); @NonNull public static final Version V6_8_0 = Objects.requireNonNull(parse("6.8.0")); + @NonNull public static final Version V6_9_0 = Objects.requireNonNull(parse("6.9.0")); private final int major; private final int minor; diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java index 995513e3919..5b0c114a5fe 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java @@ -479,7 +479,8 @@ private Optional overrideJvmVersionForDseWorkloads() { return Optional.empty(); } - if (getDseVersion().get().compareTo(Version.parse("6.8.19")) < 0) { + if (getDseVersion().get().compareTo(Version.V6_9_0) >= 0) { + // DSE 6.9.0 supports only JVM 11 onwards (also with graph workload) return Optional.empty(); } From c961012000efffd3d50476d4549487f7fc538c01 Mon Sep 17 00:00:00 2001 From: Henry Hughes Date: Wed, 23 Aug 2023 15:35:42 -0700 Subject: [PATCH 10/48] JAVA-3117: Call CcmCustomRule#after if CcmCustomRule#before fails to allow subsequent tests to run patch by Henry Hughes; reviewed by Alexandre Dutra and Andy Tolbert for JAVA-3117 --- .../api/testinfra/ccm/CustomCcmRule.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java index 58bafd438f8..cf150b12f55 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java @@ -18,6 +18,8 @@ package com.datastax.oss.driver.api.testinfra.ccm; import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A rule that creates a ccm cluster that can be used in a test. This should be used if you plan on @@ -30,6 +32,7 @@ */ public class CustomCcmRule extends BaseCcmRule { + private static final Logger LOG = LoggerFactory.getLogger(CustomCcmRule.class); private static final AtomicReference CURRENT = new AtomicReference<>(); CustomCcmRule(CcmBridge ccmBridge) { @@ -39,7 +42,20 @@ public class CustomCcmRule extends BaseCcmRule { @Override protected void before() { if (CURRENT.get() == null && CURRENT.compareAndSet(null, this)) { - super.before(); + try { + super.before(); + } catch (Exception e) { + // ExternalResource will not call after() when before() throws an exception + // Let's try and clean up and release the lock we have in CURRENT + LOG.warn( + "Error in CustomCcmRule before() method, attempting to clean up leftover state", e); + try { + after(); + } catch (Exception e1) { + LOG.warn("Error cleaning up CustomCcmRule before() failure", e1); + } + throw e; + } } else if (CURRENT.get() != this) { throw new IllegalStateException( "Attempting to use a Ccm rule while another is in use. This is disallowed"); From 77805f5103354cadb360384f4f41e0eca73d72f4 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 2 Sep 2024 06:44:53 +0200 Subject: [PATCH 11/48] JAVA-3149: Support request cancellation in request throttler patch by Lukasz Antoniak; reviewed by Andy Tolbert and Chris Lohfink for JAVA-3149 --- .../ContinuousRequestHandlerBase.java | 1 + .../core/graph/GraphRequestHandler.java | 1 + .../session/throttling/RequestThrottler.java | 8 +++++ .../internal/core/cql/CqlPrepareHandler.java | 1 + .../internal/core/cql/CqlRequestHandler.java | 1 + .../ConcurrencyLimitingRequestThrottler.java | 16 +++++++++ .../PassThroughRequestThrottler.java | 5 +++ .../RateLimitingRequestThrottler.java | 12 +++++++ ...ncurrencyLimitingRequestThrottlerTest.java | 5 +++ .../RateLimitingRequestThrottlerTest.java | 13 ++++++- .../driver/core/throttling/ThrottlingIT.java | 34 +++++++++++++++---- 11 files changed, 89 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java b/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java index 9a7be344721..0453022cb6a 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/cql/continuous/ContinuousRequestHandlerBase.java @@ -410,6 +410,7 @@ public void cancel() { cancelScheduledTasks(null); cancelGlobalTimeout(); + throttler.signalCancel(this); } private void cancelGlobalTimeout() { diff --git a/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java b/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java index 702da69b855..5c9ceb00df2 100644 --- a/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java +++ b/core/src/main/java/com/datastax/dse/driver/internal/core/graph/GraphRequestHandler.java @@ -153,6 +153,7 @@ public class GraphRequestHandler implements Throttled { try { if (t instanceof CancellationException) { cancelScheduledTasks(); + context.getRequestThrottler().signalCancel(this); } } catch (Throwable t2) { Loggers.warnWithException(LOG, "[{}] Uncaught exception", logPrefix, t2); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java index cb55fac336c..7e2b41ebbdb 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/throttling/RequestThrottler.java @@ -56,4 +56,12 @@ public interface RequestThrottler extends Closeable { * perform time-based eviction on pending requests. */ void signalTimeout(@NonNull Throttled request); + + /** + * Signals that a request has been cancelled. This indicates to the throttler that another request + * might be started. + */ + default void signalCancel(@NonNull Throttled request) { + // no-op for backward compatibility purposes + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java index 8fe1adb20b1..1ee1f303ab2 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareHandler.java @@ -124,6 +124,7 @@ protected CqlPrepareHandler( try { if (t instanceof CancellationException) { cancelTimeout(); + context.getRequestThrottler().signalCancel(this); } } catch (Throwable t2) { Loggers.warnWithException(LOG, "[{}] Uncaught exception", logPrefix, t2); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java index a1c6b0e5466..0808bdce63f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java @@ -152,6 +152,7 @@ protected CqlRequestHandler( try { if (t instanceof CancellationException) { cancelScheduledTasks(); + context.getRequestThrottler().signalCancel(this); } } catch (Throwable t2) { Loggers.warnWithException(LOG, "[{}] Uncaught exception", logPrefix, t2); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java index e8f27467c6f..438bed0953b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java @@ -145,6 +145,22 @@ public void signalTimeout(@NonNull Throttled request) { } } + @Override + public void signalCancel(@NonNull Throttled request) { + lock.lock(); + try { + if (!closed) { + if (queue.remove(request)) { // The request has been cancelled before it was active + LOG.trace("[{}] Removing cancelled request from the queue", logPrefix); + } else { + onRequestDone(); + } + } + } finally { + lock.unlock(); + } + } + @SuppressWarnings("GuardedBy") // this method is only called with the lock held private void onRequestDone() { assert lock.isHeldByCurrentThread(); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/PassThroughRequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/PassThroughRequestThrottler.java index 714c712a4e8..2210e4b26f1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/PassThroughRequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/PassThroughRequestThrottler.java @@ -69,6 +69,11 @@ public void signalTimeout(@NonNull Throttled request) { // nothing to do } + @Override + public void signalCancel(@NonNull Throttled request) { + // nothing to do + } + @Override public void close() throws IOException { // nothing to do diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottler.java index 6536804ffee..03a693dc0fe 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottler.java @@ -198,6 +198,18 @@ public void signalTimeout(@NonNull Throttled request) { } } + @Override + public void signalCancel(@NonNull Throttled request) { + lock.lock(); + try { + if (!closed && queue.remove(request)) { // The request has been cancelled before it was active + LOG.trace("[{}] Removing cancelled request from the queue", logPrefix); + } + } finally { + lock.unlock(); + } + } + @Override public void close() { lock.lock(); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java index b587ac3daa2..c01b26c1e9f 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java @@ -88,6 +88,11 @@ public void should_allow_new_request_when_active_one_times_out() { should_allow_new_request_when_active_one_completes(throttler::signalTimeout); } + @Test + public void should_allow_new_request_when_active_one_canceled() { + should_allow_new_request_when_active_one_completes(throttler::signalCancel); + } + private void should_allow_new_request_when_active_one_completes( Consumer completeCallback) { // Given diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java index 7336fb447b6..0e0fe7c1c65 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.session.throttling.Throttled; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.NettyOptions; import com.datastax.oss.driver.internal.core.util.concurrent.ScheduledTaskCapturingEventLoop; @@ -33,6 +34,7 @@ import java.time.Duration; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -164,6 +166,15 @@ public void should_reject_when_queue_is_full() { @Test public void should_remove_timed_out_request_from_queue() { + testRemoveInvalidEventFromQueue(throttler::signalTimeout); + } + + @Test + public void should_remove_cancel_request_from_queue() { + testRemoveInvalidEventFromQueue(throttler::signalCancel); + } + + private void testRemoveInvalidEventFromQueue(Consumer completeCallback) { // Given for (int i = 0; i < 5; i++) { throttler.register(new MockThrottled()); @@ -174,7 +185,7 @@ public void should_remove_timed_out_request_from_queue() { throttler.register(queued2); // When - throttler.signalTimeout(queued1); + completeCallback.accept(queued1); // Then assertThatStage(queued2.started).isNotDone(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/throttling/ThrottlingIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/throttling/ThrottlingIT.java index a6e7295eb09..6fa1a37355b 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/throttling/ThrottlingIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/throttling/ThrottlingIT.java @@ -24,13 +24,16 @@ import com.datastax.oss.driver.api.core.RequestThrottlingException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.cql.AsyncResultSet; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.api.testinfra.simulacron.SimulacronRule; import com.datastax.oss.driver.categories.ParallelizableTests; import com.datastax.oss.driver.internal.core.session.throttling.ConcurrencyLimitingRequestThrottler; import com.datastax.oss.simulacron.common.cluster.ClusterSpec; import com.datastax.oss.simulacron.common.stubbing.PrimeDsl; +import java.util.concurrent.CompletionStage; import java.util.concurrent.TimeUnit; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -39,21 +42,20 @@ public class ThrottlingIT { private static final String QUERY = "select * from foo"; + private static final int maxConcurrentRequests = 10; + private static final int maxQueueSize = 10; @Rule public SimulacronRule simulacron = new SimulacronRule(ClusterSpec.builder().withNodes(1)); - @Test - public void should_reject_request_when_throttling_by_concurrency() { + private DriverConfigLoader loader = null; + @Before + public void setUp() { // Add a delay so that requests don't complete during the test simulacron .cluster() .prime(PrimeDsl.when(QUERY).then(PrimeDsl.noRows()).delay(5, TimeUnit.SECONDS)); - - int maxConcurrentRequests = 10; - int maxQueueSize = 10; - - DriverConfigLoader loader = + loader = SessionUtils.configLoaderBuilder() .withClass( DefaultDriverOption.REQUEST_THROTTLER_CLASS, @@ -63,7 +65,10 @@ public void should_reject_request_when_throttling_by_concurrency() { maxConcurrentRequests) .withInt(DefaultDriverOption.REQUEST_THROTTLER_MAX_QUEUE_SIZE, maxQueueSize) .build(); + } + @Test + public void should_reject_request_when_throttling_by_concurrency() { try (CqlSession session = SessionUtils.newSession(simulacron, loader)) { // Saturate the session and fill the queue @@ -81,4 +86,19 @@ public void should_reject_request_when_throttling_by_concurrency() { + "(concurrent requests: 10, queue size: 10)"); } } + + @Test + public void should_propagate_cancel_to_throttler() { + try (CqlSession session = SessionUtils.newSession(simulacron, loader)) { + + // Try to saturate the session and fill the queue + for (int i = 0; i < maxConcurrentRequests + maxQueueSize; i++) { + CompletionStage future = session.executeAsync(QUERY); + future.toCompletableFuture().cancel(true); + } + + // The next query should be successful, because the previous queries were cancelled + session.execute(QUERY); + } + } } From 4fb51081a3a71b2017ddc3f43a7945e3a9d19e25 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 10 Jun 2024 12:42:29 +0200 Subject: [PATCH 12/48] Fix C* 3.0 tests failing on Jenkins patch by Lukasz Antoniak; reviewed by Bret McGuire reference: #1939 --- .../test/java/com/datastax/oss/driver/mapper/DeleteIT.java | 3 +-- .../com/datastax/oss/driver/mapper/InventoryITBase.java | 6 +++++- .../oss/driver/mapper/SelectCustomWhereClauseIT.java | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java index 0acdbeae53a..03e3597501c 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteIT.java @@ -60,8 +60,7 @@ description = ">= in WHERE clause not supported in legacy versions") public class DeleteIT extends InventoryITBase { - private static CustomCcmRule CCM_RULE = - CustomCcmRule.builder().withCassandraConfiguration("enable_sasi_indexes", "true").build(); + private static CustomCcmRule CCM_RULE = CustomCcmRule.builder().build(); private static final SessionRule SESSION_RULE = SessionRule.builder(CCM_RULE).build(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java index 2be025b3739..9495003ae49 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java @@ -59,6 +59,10 @@ public abstract class InventoryITBase { new ProductSale(MP3_DOWNLOAD.getId(), DATE_3, 7, Uuids.startOf(915192000), 0.99, 12); protected static List createStatements(BaseCcmRule ccmRule) { + return createStatements(ccmRule, false); + } + + protected static List createStatements(BaseCcmRule ccmRule, boolean requiresSasiIndex) { ImmutableList.Builder builder = ImmutableList.builder() .add( @@ -71,7 +75,7 @@ protected static List createStatements(BaseCcmRule ccmRule) { "CREATE TABLE product_sale(id uuid, day text, ts uuid, customer_id int, price " + "double, count int, PRIMARY KEY ((id, day), customer_id, ts))"); - if (supportsSASI(ccmRule) && !isSasiBroken(ccmRule)) { + if (requiresSasiIndex && supportsSASI(ccmRule) && !isSasiBroken(ccmRule)) { builder.add( "CREATE CUSTOM INDEX product_description ON product(description) " + "USING 'org.apache.cassandra.index.sasi.SASIIndex' " diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java index 3df1ccd21a7..1f1b92b8623 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/SelectCustomWhereClauseIT.java @@ -75,7 +75,7 @@ public static void setup() { SchemaChangeSynchronizer.withLock( () -> { - for (String query : createStatements(CCM_RULE)) { + for (String query : createStatements(CCM_RULE, true)) { session.execute( SimpleStatement.builder(query) .setExecutionProfile(SESSION_RULE.slowProfile()) From 6d3ba47631ebde78460168a2d33c4facde0bd731 Mon Sep 17 00:00:00 2001 From: Jason Koch Date: Mon, 12 Aug 2024 22:52:13 -0700 Subject: [PATCH 13/48] Reduce lock held duration in ConcurrencyLimitingRequestThrottler It might take some (small) time for callback handling when the throttler request proceeds to submission. Before this change, the throttler proceed request will happen while holding the lock, preventing other tasks from proceeding when there is spare capacity and even preventing tasks from enqueuing until the callback completes. By tracking the expected outcome, we can perform the callback outside of the lock. This means that request registration and submission can proceed even when a long callback is being processed. patch by Jason Koch; Reviewed by Andy Tolbert and Chris Lohfink for CASSANDRA-19922 --- .../ConcurrencyLimitingRequestThrottler.java | 39 ++++- ...ncurrencyLimitingRequestThrottlerTest.java | 143 ++++++++++++++++-- .../session/throttling/MockThrottled.java | 30 +++- .../RateLimitingRequestThrottlerTest.java | 30 ++-- 4 files changed, 206 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java index 438bed0953b..ffe0ffe9650 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottler.java @@ -25,6 +25,7 @@ import com.datastax.oss.driver.api.core.session.throttling.Throttled; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.util.ArrayDeque; import java.util.Deque; import java.util.concurrent.locks.ReentrantLock; @@ -87,6 +88,8 @@ public ConcurrencyLimitingRequestThrottler(DriverContext context) { @Override public void register(@NonNull Throttled request) { + boolean notifyReadyRequired = false; + lock.lock(); try { if (closed) { @@ -96,7 +99,7 @@ public void register(@NonNull Throttled request) { // We have capacity for one more concurrent request LOG.trace("[{}] Starting newly registered request", logPrefix); concurrentRequests += 1; - request.onThrottleReady(false); + notifyReadyRequired = true; } else if (queue.size() < maxQueueSize) { LOG.trace("[{}] Enqueuing request", logPrefix); queue.add(request); @@ -112,16 +115,26 @@ public void register(@NonNull Throttled request) { } finally { lock.unlock(); } + + // no need to hold the lock while allowing the task to progress + if (notifyReadyRequired) { + request.onThrottleReady(false); + } } @Override public void signalSuccess(@NonNull Throttled request) { + Throttled nextRequest = null; lock.lock(); try { - onRequestDone(); + nextRequest = onRequestDoneAndDequeNext(); } finally { lock.unlock(); } + + if (nextRequest != null) { + nextRequest.onThrottleReady(true); + } } @Override @@ -131,48 +144,62 @@ public void signalError(@NonNull Throttled request, @NonNull Throwable error) { @Override public void signalTimeout(@NonNull Throttled request) { + Throttled nextRequest = null; lock.lock(); try { if (!closed) { if (queue.remove(request)) { // The request timed out before it was active LOG.trace("[{}] Removing timed out request from the queue", logPrefix); } else { - onRequestDone(); + nextRequest = onRequestDoneAndDequeNext(); } } } finally { lock.unlock(); } + + if (nextRequest != null) { + nextRequest.onThrottleReady(true); + } } @Override public void signalCancel(@NonNull Throttled request) { + Throttled nextRequest = null; lock.lock(); try { if (!closed) { if (queue.remove(request)) { // The request has been cancelled before it was active LOG.trace("[{}] Removing cancelled request from the queue", logPrefix); } else { - onRequestDone(); + nextRequest = onRequestDoneAndDequeNext(); } } } finally { lock.unlock(); } + + if (nextRequest != null) { + nextRequest.onThrottleReady(true); + } } @SuppressWarnings("GuardedBy") // this method is only called with the lock held - private void onRequestDone() { + @Nullable + private Throttled onRequestDoneAndDequeNext() { assert lock.isHeldByCurrentThread(); if (!closed) { if (queue.isEmpty()) { concurrentRequests -= 1; } else { LOG.trace("[{}] Starting dequeued request", logPrefix); - queue.poll().onThrottleReady(true); // don't touch concurrentRequests since we finished one but started another + return queue.poll(); } } + + // no next task was dequeued + return null; } @Override diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java index c01b26c1e9f..7eb682070cd 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/ConcurrencyLimitingRequestThrottlerTest.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.session.throttling.Throttled; import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.function.Consumer; import org.junit.Before; import org.junit.Test; @@ -67,7 +68,7 @@ public void should_start_immediately_when_under_capacity() { throttler.register(request); // Then - assertThatStage(request.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(request.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); assertThat(throttler.getConcurrentRequests()).isEqualTo(1); assertThat(throttler.getQueue()).isEmpty(); } @@ -98,7 +99,7 @@ private void should_allow_new_request_when_active_one_completes( // Given MockThrottled first = new MockThrottled(); throttler.register(first); - assertThatStage(first.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(first.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); for (int i = 0; i < 4; i++) { // fill to capacity throttler.register(new MockThrottled()); } @@ -113,7 +114,7 @@ private void should_allow_new_request_when_active_one_completes( throttler.register(incoming); // Then - assertThatStage(incoming.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(incoming.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); assertThat(throttler.getConcurrentRequests()).isEqualTo(5); assertThat(throttler.getQueue()).isEmpty(); } @@ -132,7 +133,7 @@ public void should_enqueue_when_over_capacity() { throttler.register(incoming); // Then - assertThatStage(incoming.started).isNotDone(); + assertThatStage(incoming.ended).isNotDone(); assertThat(throttler.getConcurrentRequests()).isEqualTo(5); assertThat(throttler.getQueue()).containsExactly(incoming); } @@ -157,20 +158,20 @@ private void should_dequeue_when_active_completes(Consumer completeCa // Given MockThrottled first = new MockThrottled(); throttler.register(first); - assertThatStage(first.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(first.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); for (int i = 0; i < 4; i++) { throttler.register(new MockThrottled()); } MockThrottled incoming = new MockThrottled(); throttler.register(incoming); - assertThatStage(incoming.started).isNotDone(); + assertThatStage(incoming.ended).isNotDone(); // When completeCallback.accept(first); // Then - assertThatStage(incoming.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); + assertThatStage(incoming.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); assertThat(throttler.getConcurrentRequests()).isEqualTo(5); assertThat(throttler.getQueue()).isEmpty(); } @@ -189,7 +190,7 @@ public void should_reject_when_queue_is_full() { throttler.register(incoming); // Then - assertThatStage(incoming.started) + assertThatStage(incoming.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } @@ -208,7 +209,7 @@ public void should_remove_timed_out_request_from_queue() { throttler.signalTimeout(queued1); // Then - assertThatStage(queued2.started).isNotDone(); + assertThatStage(queued2.ended).isNotDone(); assertThat(throttler.getConcurrentRequests()).isEqualTo(5); assertThat(throttler.getQueue()).hasSize(1); } @@ -223,7 +224,7 @@ public void should_reject_enqueued_when_closing() { for (int i = 0; i < 10; i++) { MockThrottled request = new MockThrottled(); throttler.register(request); - assertThatStage(request.started).isNotDone(); + assertThatStage(request.ended).isNotDone(); enqueued.add(request); } @@ -232,7 +233,7 @@ public void should_reject_enqueued_when_closing() { // Then for (MockThrottled request : enqueued) { - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } @@ -241,7 +242,125 @@ public void should_reject_enqueued_when_closing() { throttler.register(request); // Then - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } + + @Test + public void should_run_throttle_callbacks_concurrently() throws InterruptedException { + // Given + + // a task is enqueued, which when in onThrottleReady, will stall latch countDown()ed + // register() should automatically start onThrottleReady on same thread + + // start a parallel thread + CountDownLatch firstRelease = new CountDownLatch(1); + MockThrottled first = new MockThrottled(firstRelease); + Runnable r = + () -> { + throttler.register(first); + first.ended.toCompletableFuture().thenRun(() -> throttler.signalSuccess(first)); + }; + Thread t = new Thread(r); + t.start(); + + // wait for the registration threads to reach await state + assertThatStage(first.started).isSuccess(); + assertThatStage(first.ended).isNotDone(); + + // When + // we concurrently submit a second shorter task + MockThrottled second = new MockThrottled(); + // (on a second thread, so that we can join and force a timeout in case + // registration is delayed) + Thread t2 = new Thread(() -> throttler.register(second)); + t2.start(); + t2.join(1_000); + + // Then + // registration will trigger callback, should complete ~immediately + assertThatStage(second.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + // first should still be unfinished + assertThatStage(first.started).isDone(); + assertThatStage(first.ended).isNotDone(); + // now finish, and verify + firstRelease.countDown(); + assertThatStage(first.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + + t.join(1_000); + } + + @Test + public void should_enqueue_tasks_quickly_when_callbacks_blocked() throws InterruptedException { + // Given + + // Multiple tasks are registered, up to the limit, and proceed into their + // callback + + // start five parallel threads + final int THREADS = 5; + Thread[] threads = new Thread[THREADS]; + CountDownLatch[] latches = new CountDownLatch[THREADS]; + MockThrottled[] throttled = new MockThrottled[THREADS]; + for (int i = 0; i < threads.length; i++) { + latches[i] = new CountDownLatch(1); + final MockThrottled itThrottled = new MockThrottled(latches[i]); + throttled[i] = itThrottled; + threads[i] = + new Thread( + () -> { + throttler.register(itThrottled); + itThrottled + .ended + .toCompletableFuture() + .thenRun(() -> throttler.signalSuccess(itThrottled)); + }); + threads[i].start(); + } + + // wait for the registration threads to be launched + // they are all waiting now + for (int i = 0; i < throttled.length; i++) { + assertThatStage(throttled[i].started).isSuccess(); + assertThatStage(throttled[i].ended).isNotDone(); + } + + // When + // we concurrently submit another task + MockThrottled last = new MockThrottled(); + throttler.register(last); + + // Then + // registration will enqueue the callback, and it should not + // take any time to proceed (ie: we should not be blocked) + // and there should be an element in the queue + assertThatStage(last.started).isNotDone(); + assertThatStage(last.ended).isNotDone(); + assertThat(throttler.getQueue()).containsExactly(last); + + // we still have not released, so old throttled threads should be waiting + for (int i = 0; i < throttled.length; i++) { + assertThatStage(throttled[i].started).isDone(); + assertThatStage(throttled[i].ended).isNotDone(); + } + + // now let us release .. + for (int i = 0; i < latches.length; i++) { + latches[i].countDown(); + } + + // .. and check everything finished up OK + for (int i = 0; i < latches.length; i++) { + assertThatStage(throttled[i].started).isSuccess(); + assertThatStage(throttled[i].ended).isSuccess(); + } + + // for good measure, we will also wait for the enqueued to complete + assertThatStage(last.started).isSuccess(); + assertThatStage(last.ended).isSuccess(); + + for (int i = 0; i < threads.length; i++) { + threads[i].join(1_000); + } + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/MockThrottled.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/MockThrottled.java index b7cd0ee8a54..9e54e3d511f 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/MockThrottled.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/MockThrottled.java @@ -19,21 +19,45 @@ import com.datastax.oss.driver.api.core.RequestThrottlingException; import com.datastax.oss.driver.api.core.session.throttling.Throttled; +import com.datastax.oss.driver.shaded.guava.common.util.concurrent.Uninterruptibles; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; class MockThrottled implements Throttled { + final CompletionStage started = new CompletableFuture<>(); + final CompletionStage ended = new CompletableFuture<>(); + final CountDownLatch canRelease; - final CompletionStage started = new CompletableFuture<>(); + public MockThrottled() { + this(new CountDownLatch(0)); + } + + /* + * The releaseLatch can be provided to add some delay before the + * task readiness/fail callbacks complete. This can be used, eg, to + * imitate a slow callback. + */ + public MockThrottled(CountDownLatch releaseLatch) { + this.canRelease = releaseLatch; + } @Override public void onThrottleReady(boolean wasDelayed) { - started.toCompletableFuture().complete(wasDelayed); + started.toCompletableFuture().complete(null); + awaitRelease(); + ended.toCompletableFuture().complete(wasDelayed); } @Override public void onThrottleFailure(@NonNull RequestThrottlingException error) { - started.toCompletableFuture().completeExceptionally(error); + started.toCompletableFuture().complete(null); + awaitRelease(); + ended.toCompletableFuture().completeExceptionally(error); + } + + private void awaitRelease() { + Uninterruptibles.awaitUninterruptibly(canRelease); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java index 0e0fe7c1c65..1e15610bf7b 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/session/throttling/RateLimitingRequestThrottlerTest.java @@ -98,7 +98,7 @@ public void should_start_immediately_when_under_capacity() { throttler.register(request); // Then - assertThatStage(request.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(request.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); assertThat(throttler.getStoredPermits()).isEqualTo(4); assertThat(throttler.getQueue()).isEmpty(); } @@ -117,7 +117,7 @@ public void should_allow_new_request_when_under_rate() { throttler.register(request); // Then - assertThatStage(request.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); + assertThatStage(request.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isFalse()); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).isEmpty(); } @@ -136,7 +136,7 @@ public void should_enqueue_when_over_rate() { throttler.register(request); // Then - assertThatStage(request.started).isNotDone(); + assertThatStage(request.ended).isNotDone(); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).containsExactly(request); @@ -160,7 +160,7 @@ public void should_reject_when_queue_is_full() { throttler.register(request); // Then - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } @@ -188,7 +188,7 @@ private void testRemoveInvalidEventFromQueue(Consumer completeCallbac completeCallback.accept(queued1); // Then - assertThatStage(queued2.started).isNotDone(); + assertThatStage(queued2.ended).isNotDone(); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).containsExactly(queued2); } @@ -202,10 +202,10 @@ public void should_dequeue_when_draining_task_runs() { MockThrottled queued1 = new MockThrottled(); throttler.register(queued1); - assertThatStage(queued1.started).isNotDone(); + assertThatStage(queued1.ended).isNotDone(); MockThrottled queued2 = new MockThrottled(); throttler.register(queued2); - assertThatStage(queued2.started).isNotDone(); + assertThatStage(queued2.ended).isNotDone(); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).hasSize(2); @@ -230,8 +230,8 @@ public void should_dequeue_when_draining_task_runs() { task.run(); // Then - assertThatStage(queued1.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); - assertThatStage(queued2.started).isNotDone(); + assertThatStage(queued1.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); + assertThatStage(queued2.ended).isNotDone(); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).containsExactly(queued2); // task reschedules itself since it did not empty the queue @@ -244,7 +244,7 @@ public void should_dequeue_when_draining_task_runs() { task.run(); // Then - assertThatStage(queued2.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); + assertThatStage(queued2.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); assertThat(throttler.getStoredPermits()).isEqualTo(0); assertThat(throttler.getQueue()).isEmpty(); assertThat(adminExecutor.nextTask()).isNull(); @@ -286,14 +286,14 @@ public void should_keep_accumulating_time_if_no_permits_created() { // Then MockThrottled queued = new MockThrottled(); throttler.register(queued); - assertThatStage(queued.started).isNotDone(); + assertThatStage(queued.ended).isNotDone(); // When clock.add(ONE_HUNDRED_MILLISECONDS); adminExecutor.nextTask().run(); // Then - assertThatStage(queued.started).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); + assertThatStage(queued.ended).isSuccess(wasDelayed -> assertThat(wasDelayed).isTrue()); } @Test @@ -306,7 +306,7 @@ public void should_reject_enqueued_when_closing() { for (int i = 0; i < 10; i++) { MockThrottled request = new MockThrottled(); throttler.register(request); - assertThatStage(request.started).isNotDone(); + assertThatStage(request.ended).isNotDone(); enqueued.add(request); } @@ -315,7 +315,7 @@ public void should_reject_enqueued_when_closing() { // Then for (MockThrottled request : enqueued) { - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } @@ -324,7 +324,7 @@ public void should_reject_enqueued_when_closing() { throttler.register(request); // Then - assertThatStage(request.started) + assertThatStage(request.ended) .isFailed(error -> assertThat(error).isInstanceOf(RequestThrottlingException.class)); } } From 306bf3744a3d24ad01700e4e1d3c14b9a696927f Mon Sep 17 00:00:00 2001 From: Ammar Khaku Date: Fri, 30 Sep 2022 15:36:36 -0700 Subject: [PATCH 14/48] Annotate BatchStatement, Statement, SimpleStatement methods with CheckReturnValue Since the driver's default implementation is for BatchStatement and SimpleStatement methods to be immutable, we should annotate those methods with @CheckReturnValue. Statement#setNowInSeconds implementations are immutable so annotate that too. patch by Ammar Khaku; reviewed by Andy Tolbert and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1607 --- core/revapi.json | 133 ++++++++++++++++++ .../driver/api/core/cql/BatchStatement.java | 8 ++ .../driver/api/core/cql/SimpleStatement.java | 7 + .../oss/driver/api/core/cql/Statement.java | 1 + 4 files changed, 149 insertions(+) diff --git a/core/revapi.json b/core/revapi.json index 318e29709ec..1c875895d6c 100644 --- a/core/revapi.json +++ b/core/revapi.json @@ -6956,6 +6956,139 @@ "old": "method java.lang.Throwable java.lang.Throwable::fillInStackTrace() @ com.fasterxml.jackson.databind.deser.UnresolvedForwardReference", "new": "method com.fasterxml.jackson.databind.deser.UnresolvedForwardReference com.fasterxml.jackson.databind.deser.UnresolvedForwardReference::fillInStackTrace()", "justification": "Upgrade jackson-databind to 2.13.4.1 to address CVEs, API change cause: https://github.com/FasterXML/jackson-databind/issues/3419" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchStatement", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchStatement", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchableStatement>", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchStatement", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchableStatement>", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BatchableStatement>", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BoundStatement", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.BoundStatement", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.SimpleStatement", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int) @ com.datastax.oss.driver.api.core.cql.SimpleStatement", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int)", + "new": "method SelfT com.datastax.oss.driver.api.core.cql.Statement>::setNowInSeconds(int)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::add(com.datastax.oss.driver.api.core.cql.BatchableStatement)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::add(com.datastax.oss.driver.api.core.cql.BatchableStatement)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::addAll(com.datastax.oss.driver.api.core.cql.BatchableStatement[])", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::addAll(com.datastax.oss.driver.api.core.cql.BatchableStatement[])", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::addAll(java.lang.Iterable>)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::addAll(java.lang.Iterable>)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::clear()", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::clear()", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setBatchType(com.datastax.oss.driver.api.core.cql.BatchType)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setBatchType(com.datastax.oss.driver.api.core.cql.BatchType)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setKeyspace(com.datastax.oss.driver.api.core.CqlIdentifier)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setKeyspace(com.datastax.oss.driver.api.core.CqlIdentifier)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "JAVA-2161: Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setKeyspace(java.lang.String)", + "new": "method com.datastax.oss.driver.api.core.cql.BatchStatement com.datastax.oss.driver.api.core.cql.BatchStatement::setKeyspace(java.lang.String)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "JAVA-2161: Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setQuery(java.lang.String)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setQuery(java.lang.String)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setKeyspace(com.datastax.oss.driver.api.core.CqlIdentifier)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setKeyspace(com.datastax.oss.driver.api.core.CqlIdentifier)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setKeyspace(java.lang.String)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setKeyspace(java.lang.String)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setPositionalValues(java.util.List)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setPositionalValues(java.util.List)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValuesWithIds(java.util.Map)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValuesWithIds(java.util.Map)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.annotation.added", + "old": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValues(java.util.Map)", + "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValues(java.util.Map)", + "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", + "justification": "Annotate mutating methods with @CheckReturnValue" } ] } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/cql/BatchStatement.java b/core/src/main/java/com/datastax/oss/driver/api/core/cql/BatchStatement.java index 0f37ed71ce2..9deb33c6007 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/cql/BatchStatement.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/cql/BatchStatement.java @@ -26,6 +26,7 @@ import com.datastax.oss.driver.internal.core.util.Sizes; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.protocol.internal.PrimitiveSizes; +import edu.umd.cs.findbugs.annotations.CheckReturnValue; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.util.ArrayList; @@ -164,6 +165,7 @@ static BatchStatementBuilder builder(@NonNull BatchStatement template) { * method. However custom implementations may choose to be mutable and return the same instance. */ @NonNull + @CheckReturnValue BatchStatement setBatchType(@NonNull BatchType newBatchType); /** @@ -180,6 +182,7 @@ static BatchStatementBuilder builder(@NonNull BatchStatement template) { * @see Request#getKeyspace() */ @NonNull + @CheckReturnValue BatchStatement setKeyspace(@Nullable CqlIdentifier newKeyspace); /** @@ -187,6 +190,7 @@ static BatchStatementBuilder builder(@NonNull BatchStatement template) { * setKeyspace(CqlIdentifier.fromCql(newKeyspaceName))}. */ @NonNull + @CheckReturnValue default BatchStatement setKeyspace(@NonNull String newKeyspaceName) { return setKeyspace(CqlIdentifier.fromCql(newKeyspaceName)); } @@ -201,6 +205,7 @@ default BatchStatement setKeyspace(@NonNull String newKeyspaceName) { * method. However custom implementations may choose to be mutable and return the same instance. */ @NonNull + @CheckReturnValue BatchStatement add(@NonNull BatchableStatement statement); /** @@ -213,10 +218,12 @@ default BatchStatement setKeyspace(@NonNull String newKeyspaceName) { * method. However custom implementations may choose to be mutable and return the same instance. */ @NonNull + @CheckReturnValue BatchStatement addAll(@NonNull Iterable> statements); /** @see #addAll(Iterable) */ @NonNull + @CheckReturnValue default BatchStatement addAll(@NonNull BatchableStatement... statements) { return addAll(Arrays.asList(statements)); } @@ -231,6 +238,7 @@ default BatchStatement addAll(@NonNull BatchableStatement... statements) { * method. However custom implementations may choose to be mutable and return the same instance. */ @NonNull + @CheckReturnValue BatchStatement clear(); @Override diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/cql/SimpleStatement.java b/core/src/main/java/com/datastax/oss/driver/api/core/cql/SimpleStatement.java index fd5f456f11c..ef04cd14a5b 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/cql/SimpleStatement.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/cql/SimpleStatement.java @@ -28,6 +28,7 @@ import com.datastax.oss.protocol.internal.PrimitiveSizes; import com.datastax.oss.protocol.internal.util.collection.NullAllowingImmutableList; import com.datastax.oss.protocol.internal.util.collection.NullAllowingImmutableMap; +import edu.umd.cs.findbugs.annotations.CheckReturnValue; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.util.List; @@ -197,6 +198,7 @@ static SimpleStatementBuilder builder(@NonNull SimpleStatement template) { * @see #setNamedValuesWithIds(Map) */ @NonNull + @CheckReturnValue SimpleStatement setQuery(@NonNull String newQuery); /** @@ -209,6 +211,7 @@ static SimpleStatementBuilder builder(@NonNull SimpleStatement template) { * @see Request#getKeyspace() */ @NonNull + @CheckReturnValue SimpleStatement setKeyspace(@Nullable CqlIdentifier newKeyspace); /** @@ -216,6 +219,7 @@ static SimpleStatementBuilder builder(@NonNull SimpleStatement template) { * setKeyspace(CqlIdentifier.fromCql(newKeyspaceName))}. */ @NonNull + @CheckReturnValue default SimpleStatement setKeyspace(@NonNull String newKeyspaceName) { return setKeyspace(CqlIdentifier.fromCql(newKeyspaceName)); } @@ -236,6 +240,7 @@ default SimpleStatement setKeyspace(@NonNull String newKeyspaceName) { * @see #setQuery(String) */ @NonNull + @CheckReturnValue SimpleStatement setPositionalValues(@NonNull List newPositionalValues); @NonNull @@ -256,6 +261,7 @@ default SimpleStatement setKeyspace(@NonNull String newKeyspaceName) { * @see #setQuery(String) */ @NonNull + @CheckReturnValue SimpleStatement setNamedValuesWithIds(@NonNull Map newNamedValues); /** @@ -263,6 +269,7 @@ default SimpleStatement setKeyspace(@NonNull String newKeyspaceName) { * converted on the fly with {@link CqlIdentifier#fromCql(String)}. */ @NonNull + @CheckReturnValue default SimpleStatement setNamedValues(@NonNull Map newNamedValues) { return setNamedValuesWithIds(DefaultSimpleStatement.wrapKeys(newNamedValues)); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/cql/Statement.java b/core/src/main/java/com/datastax/oss/driver/api/core/cql/Statement.java index 594c627e324..d70c56686c5 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/cql/Statement.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/cql/Statement.java @@ -513,6 +513,7 @@ default int getNowInSeconds() { * @see #NO_NOW_IN_SECONDS */ @NonNull + @CheckReturnValue @SuppressWarnings("unchecked") default SelfT setNowInSeconds(int nowInSeconds) { return (SelfT) this; From 8444c79ff843a072e5c1a1d8de5140a47051e2a0 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Mon, 16 Sep 2024 16:30:43 -0500 Subject: [PATCH 15/48] Remove "beta" support for Java17 from docs patch by Bret McGuire; reviewed by Andy Tolbert and Alexandre Dutra reference: https://github.com/apache/cassandra-java-driver/pull/1962 --- upgrade_guide/README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index c6df74ffc2a..56d55aaab36 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -19,7 +19,7 @@ under the License. ## Upgrade guide -### NEW VERSION PLACEHOLDER +### 4.18.1 #### Keystore reloading in DefaultSslEngineFactory @@ -32,12 +32,9 @@ This feature is disabled by default for compatibility. To enable, see `keystore- ### 4.17.0 -#### Beta support for Java17 +#### Support for Java17 With the completion of [JAVA-3042](https://datastax-oss.atlassian.net/browse/JAVA-3042) the driver now passes our automated test matrix for Java Driver releases. -While all features function normally when run with Java 17 tests, we do not offer full support for this -platform until we've received feedback from other users in the ecosystem. - If you discover an issue with the Java Driver running on Java 17, please let us know. We will triage and address Java 17 issues. #### Updated API for vector search From a40e7587b175cc198fb533eadabd31e94f837369 Mon Sep 17 00:00:00 2001 From: Christian Aistleitner Date: Thu, 6 Jun 2024 09:14:16 +0200 Subject: [PATCH 16/48] Fix uncaught exception during graceful channel shutdown after exceeding max orphan ids patch by Christian Aistleitner; reviewed by Andy Tolbert, and Bret McGuire for #1938 --- .../core/channel/InFlightHandler.java | 4 +- .../core/channel/InFlightHandlerTest.java | 63 ++++++++++++++++++- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/InFlightHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/InFlightHandler.java index 9060f80b7cd..90b02f358cd 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/InFlightHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/InFlightHandler.java @@ -199,14 +199,14 @@ private void startGracefulShutdown(ChannelHandlerContext ctx) { LOG.debug("[{}] No pending queries, completing graceful shutdown now", logPrefix); ctx.channel().close(); } else { - // remove heartbeat handler from pipeline if present. + // Remove heartbeat handler from pipeline if present. ChannelHandler heartbeatHandler = ctx.pipeline().get(ChannelFactory.HEARTBEAT_HANDLER_NAME); if (heartbeatHandler != null) { ctx.pipeline().remove(heartbeatHandler); } LOG.debug("[{}] There are pending queries, delaying graceful shutdown", logPrefix); closingGracefully = true; - closeStartedFuture.setSuccess(); + closeStartedFuture.trySuccess(); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/InFlightHandlerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/InFlightHandlerTest.java index 79a575d9eb6..35049e99af1 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/channel/InFlightHandlerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/channel/InFlightHandlerTest.java @@ -39,7 +39,9 @@ import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelPromise; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import org.junit.Before; import org.junit.Test; @@ -256,7 +258,7 @@ public void should_refuse_new_writes_during_graceful_close() { } @Test - public void should_close_gracefully_if_orphan_ids_above_max_and_pending_requests() { + public void should_close_gracefully_if_orphan_ids_above_max_and_pending_request() { // Given addToPipeline(); // Generate n orphan ids by writing and cancelling the requests: @@ -311,6 +313,65 @@ public void should_close_gracefully_if_orphan_ids_above_max_and_pending_requests assertThat(channel.closeFuture()).isSuccess(); } + @Test + public void should_close_gracefully_if_orphan_ids_above_max_and_multiple_pending_requests() { + // Given + addToPipeline(); + // Generate n orphan ids by writing and cancelling the requests. + for (int i = 0; i < MAX_ORPHAN_IDS; i++) { + when(streamIds.acquire()).thenReturn(i); + MockResponseCallback responseCallback = new MockResponseCallback(); + channel + .writeAndFlush( + new DriverChannel.RequestMessage(QUERY, false, Frame.NO_PAYLOAD, responseCallback)) + .awaitUninterruptibly(); + channel.writeAndFlush(responseCallback).awaitUninterruptibly(); + } + // Generate 3 additional requests that are pending and not cancelled. + List pendingResponseCallbacks = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + when(streamIds.acquire()).thenReturn(MAX_ORPHAN_IDS + i); + MockResponseCallback responseCallback = new MockResponseCallback(); + channel + .writeAndFlush( + new DriverChannel.RequestMessage(QUERY, false, Frame.NO_PAYLOAD, responseCallback)) + .awaitUninterruptibly(); + pendingResponseCallbacks.add(responseCallback); + } + + // When + // Generate the n+1th orphan id that makes us go above the threshold by canceling one if the + // pending requests. + channel.writeAndFlush(pendingResponseCallbacks.remove(0)).awaitUninterruptibly(); + + // Then + // Channel should be closing gracefully but there's no way to observe that from the outside + // besides writing another request and check that it's rejected. + assertThat(channel.closeFuture()).isNotDone(); + ChannelFuture otherWriteFuture = + channel.writeAndFlush( + new DriverChannel.RequestMessage( + QUERY, false, Frame.NO_PAYLOAD, new MockResponseCallback())); + assertThat(otherWriteFuture).isFailed(); + assertThat(otherWriteFuture.cause()) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Channel is closing"); + + // When + // Cancel the remaining pending requests causing the n+ith orphan ids above the threshold. + for (MockResponseCallback pendingResponseCallback : pendingResponseCallbacks) { + ChannelFuture future = channel.writeAndFlush(pendingResponseCallback).awaitUninterruptibly(); + + // Then + // The future should succeed even though the channel has started closing gracefully. + assertThat(future).isSuccess(); + } + + // Then + // The graceful shutdown completes. + assertThat(channel.closeFuture()).isSuccess(); + } + @Test public void should_close_immediately_if_orphan_ids_above_max_and_no_pending_requests() { // Given From eb57fd7d46fb8b84655e66a3ca8e9ceef77b5164 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 18 Sep 2024 08:49:37 +0000 Subject: [PATCH 17/48] Build a public CI for Apache Cassandra Java Driver patch by Siyao (Jane) He; reviewed by Mick Semb Wever for CASSANDRA-19832 --- Jenkinsfile-asf | 80 +++++++++++++++++++++++++++++ Jenkinsfile => Jenkinsfile-datastax | 0 ci/create-user.sh | 60 ++++++++++++++++++++++ ci/run-tests.sh | 10 ++++ 4 files changed, 150 insertions(+) create mode 100644 Jenkinsfile-asf rename Jenkinsfile => Jenkinsfile-datastax (100%) create mode 100644 ci/create-user.sh create mode 100755 ci/run-tests.sh diff --git a/Jenkinsfile-asf b/Jenkinsfile-asf new file mode 100644 index 00000000000..a1be4bcd4f6 --- /dev/null +++ b/Jenkinsfile-asf @@ -0,0 +1,80 @@ +#!groovy + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +pipeline { + agent { + label 'cassandra-small' + } + + triggers { + // schedules only run against release branches (i.e. 3.x, 4.x, 4.5.x, etc.) + cron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? '@weekly' : '') + } + + stages { + stage('Matrix') { + matrix { + axes { + axis { + name 'TEST_JAVA_VERSION' + values 'openjdk@1.8.0-292', 'openjdk@1.11.0-9', 'openjdk@17' + } + axis { + name 'SERVER_VERSION' + values '3.11.17', + '4.0.13', + '5.0-beta1' + } + } + stages { + stage('Tests') { + agent { + label 'cassandra-medium' + } + steps { + script { + executeTests() + junit testResults: '**/target/surefire-reports/TEST-*.xml', allowEmptyResults: true + junit testResults: '**/target/failsafe-reports/TEST-*.xml', allowEmptyResults: true + } + } + } + } + } + } + } +} + +def executeTests() { + def testJavaMajorVersion = (TEST_JAVA_VERSION =~ /@(?:1\.)?(\d+)/)[0][1] + sh """ + container_id=\$(docker run -td -e TEST_JAVA_VERSION=${TEST_JAVA_VERSION} -e SERVER_VERSION=${SERVER_VERSION} -e TEST_JAVA_MAJOR_VERSION=${testJavaMajorVersion} -v \$(pwd):/home/docker/cassandra-java-driver apache.jfrog.io/cassan-docker/apache/cassandra-java-driver-testing-ubuntu2204 'sleep 2h') + docker exec --user root \$container_id bash -c \"sudo bash /home/docker/cassandra-java-driver/ci/create-user.sh docker \$(id -u) \$(id -g) /home/docker/cassandra-java-driver\" + docker exec --user docker \$container_id './cassandra-java-driver/ci/run-tests.sh' + ( nohup docker stop \$container_id >/dev/null 2>/dev/null & ) + """ +} + +// branch pattern for cron +// should match 3.x, 4.x, 4.5.x, etc +def branchPatternCron() { + ~'((\\d+(\\.[\\dx]+)+))' +} diff --git a/Jenkinsfile b/Jenkinsfile-datastax similarity index 100% rename from Jenkinsfile rename to Jenkinsfile-datastax diff --git a/ci/create-user.sh b/ci/create-user.sh new file mode 100644 index 00000000000..fb193df9a00 --- /dev/null +++ b/ci/create-user.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################ +# +# Prep +# +################################ + +if [ "$1" == "-h" ]; then + echo "$0 [-h] " + echo " this script is used internally by other scripts in the same directory to create a user with the running host user's same uid and gid" + exit 1 +fi + +# arguments +username=$1 +uid=$2 +gid=$3 +BUILD_HOME=$4 + +################################ +# +# Main +# +################################ + +# disable git directory ownership checks +su ${username} -c "git config --global safe.directory '*'" + +if grep "^ID=" /etc/os-release | grep -q 'debian\|ubuntu' ; then + deluser docker + adduser --quiet --disabled-login --no-create-home --uid $uid --gecos ${username} ${username} + groupmod --non-unique -g $gid $username + gpasswd -a ${username} sudo >/dev/null +else + adduser --no-create-home --uid $uid ${username} +fi + +# sudo priviledges +echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username} +chmod 0440 /etc/sudoers.d/${username} + +# proper permissions +chown -R ${username}:${username} /home/docker +chmod og+wx ${BUILD_HOME} \ No newline at end of file diff --git a/ci/run-tests.sh b/ci/run-tests.sh new file mode 100755 index 00000000000..02a8070e7a9 --- /dev/null +++ b/ci/run-tests.sh @@ -0,0 +1,10 @@ +#!/bin/bash -x + +. ~/.jabba/jabba.sh +. ~/env.txt +cd $(dirname "$(readlink -f "$0")")/.. +printenv | sort +mvn -B -V install -DskipTests -Dmaven.javadoc.skip=true +jabba use ${TEST_JAVA_VERSION} +printenv | sort +mvn -B -V verify -T 1 -Ptest-jdk-${TEST_JAVA_MAJOR_VERSION} -DtestJavaHome=$(jabba which ${TEST_JAVA_VERSION}) -Dccm.version=${SERVER_VERSION} -Dccm.dse=false -Dmaven.test.failure.ignore=true -Dmaven.javadoc.skip=true From 72c729b1ba95695fed467ca3734de7a39a2b3201 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Thu, 3 Oct 2024 09:32:17 +0200 Subject: [PATCH 18/48] CASSANDRA-19932: Allow to define extensions while creating table patch by Lukasz Antoniak; reviewed by Bret McGuire and Chris Lohfink --- .../schema/CreateTableWithOptions.java | 11 +++++ .../schema/RawOptionsWrapper.java | 45 +++++++++++++++++++ .../querybuilder/schema/CreateTableTest.java | 9 +++- 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/schema/RawOptionsWrapper.java diff --git a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableWithOptions.java b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableWithOptions.java index 4dd3193da15..c7bddf575fb 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableWithOptions.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableWithOptions.java @@ -18,7 +18,11 @@ package com.datastax.oss.driver.api.querybuilder.schema; import com.datastax.oss.driver.api.querybuilder.BuildableQuery; +import com.datastax.oss.driver.internal.querybuilder.schema.RawOptionsWrapper; +import com.datastax.oss.driver.shaded.guava.common.collect.Maps; +import edu.umd.cs.findbugs.annotations.CheckReturnValue; import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Map; public interface CreateTableWithOptions extends BuildableQuery, RelationStructure { @@ -26,4 +30,11 @@ public interface CreateTableWithOptions /** Enables COMPACT STORAGE in the CREATE TABLE statement. */ @NonNull CreateTableWithOptions withCompactStorage(); + + /** Attaches custom metadata to CQL table definition. */ + @NonNull + @CheckReturnValue + default CreateTableWithOptions withExtensions(@NonNull Map extensions) { + return withOption("extensions", Maps.transformValues(extensions, RawOptionsWrapper::of)); + } } diff --git a/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/schema/RawOptionsWrapper.java b/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/schema/RawOptionsWrapper.java new file mode 100644 index 00000000000..64cdb50f887 --- /dev/null +++ b/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/schema/RawOptionsWrapper.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.querybuilder.schema; + +import com.datastax.oss.driver.api.core.data.ByteUtils; + +/** + * Wrapper class to indicate that the contained String value should be understood to represent a CQL + * literal that can be included directly in a CQL statement (i.e. without escaping). + */ +public class RawOptionsWrapper { + private final String val; + + private RawOptionsWrapper(String val) { + this.val = val; + } + + public static RawOptionsWrapper of(String val) { + return new RawOptionsWrapper(val); + } + + public static RawOptionsWrapper of(byte[] val) { + return new RawOptionsWrapper(ByteUtils.toHexString(val)); + } + + @Override + public String toString() { + return this.val; + } +} diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java index d32c66f629b..7a5542c51f0 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java @@ -28,6 +28,7 @@ import com.datastax.oss.driver.api.querybuilder.schema.compaction.TimeWindowCompactionStrategy.CompactionWindowUnit; import com.datastax.oss.driver.api.querybuilder.schema.compaction.TimeWindowCompactionStrategy.TimestampResolution; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import java.nio.charset.StandardCharsets; import org.junit.Test; public class CreateTableTest { @@ -169,6 +170,12 @@ public void should_generate_create_table_with_options() { .withComment("Hello world") .withDcLocalReadRepairChance(0.54) .withDefaultTimeToLiveSeconds(86400) + .withExtensions( + ImmutableMap.of( + "key1", + "apache".getBytes(StandardCharsets.UTF_8), + "key2", + "cassandra".getBytes(StandardCharsets.UTF_8))) .withGcGraceSeconds(864000) .withMemtableFlushPeriodInMs(10000) .withMinIndexInterval(1024) @@ -176,7 +183,7 @@ public void should_generate_create_table_with_options() { .withReadRepairChance(0.55) .withSpeculativeRetry("99percentile")) .hasCql( - "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH bloom_filter_fp_chance=0.42 AND cdc=false AND comment='Hello world' AND dclocal_read_repair_chance=0.54 AND default_time_to_live=86400 AND gc_grace_seconds=864000 AND memtable_flush_period_in_ms=10000 AND min_index_interval=1024 AND max_index_interval=4096 AND read_repair_chance=0.55 AND speculative_retry='99percentile'"); + "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH bloom_filter_fp_chance=0.42 AND cdc=false AND comment='Hello world' AND dclocal_read_repair_chance=0.54 AND default_time_to_live=86400 AND extensions={'key1':0x617061636865,'key2':0x63617373616e647261} AND gc_grace_seconds=864000 AND memtable_flush_period_in_ms=10000 AND min_index_interval=1024 AND max_index_interval=4096 AND read_repair_chance=0.55 AND speculative_retry='99percentile'"); } @Test From 8ebcd9f85afb548f38e953fb1190d9ff04d8df5a Mon Sep 17 00:00:00 2001 From: Abe Ratnofsky Date: Tue, 15 Oct 2024 19:24:00 -0400 Subject: [PATCH 19/48] Fix DefaultSslEngineFactory missing null check on close patch by Abe Ratnofsky; reviewed by Andy Tolbert and Chris Lohfink for CASSANDRA-20001 --- .../oss/driver/internal/core/ssl/DefaultSslEngineFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java index bb95dc738c7..475ec38d578 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/ssl/DefaultSslEngineFactory.java @@ -164,6 +164,6 @@ private ReloadingKeyManagerFactory buildReloadingKeyManagerFactory(DriverExecuti @Override public void close() throws Exception { - kmf.close(); + if (kmf != null) kmf.close(); } } From f98e3433b91b49e0facfbce8e94e01e304714968 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Wed, 2 Oct 2024 18:04:19 -0500 Subject: [PATCH 20/48] Query builder support for NOT CQL syntax patch by Bret McGuire; reviewed by Bret McGuire and Andy Tolbert for CASSANDRA-19930 --- .../relation/ColumnRelationBuilder.java | 24 +++++++ .../relation/InRelationBuilder.java | 32 +++++++++ .../querybuilder/relation/RelationTest.java | 69 ++++++++++++++++++- 3 files changed, 124 insertions(+), 1 deletion(-) diff --git a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/ColumnRelationBuilder.java b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/ColumnRelationBuilder.java index 613e72291b7..247d61eaed5 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/ColumnRelationBuilder.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/ColumnRelationBuilder.java @@ -46,4 +46,28 @@ default ResultT contains(@NonNull Term term) { default ResultT containsKey(@NonNull Term term) { return build(" CONTAINS KEY ", term); } + + /** + * Builds a NOT CONTAINS relation for the column. + * + *

Note that NOT CONTAINS support is only available in Cassandra 5.1 or later. See CASSANDRA-18584 for more + * information. + */ + @NonNull + default ResultT notContains(@NonNull Term term) { + return build(" NOT CONTAINS ", term); + } + + /** + * Builds a NOT CONTAINS KEY relation for the column. + * + *

Note that NOT CONTAINS KEY support is only available in Cassandra 5.1 or later. See CASSANDRA-18584 for more + * information. + */ + @NonNull + default ResultT notContainsKey(@NonNull Term term) { + return build(" NOT CONTAINS KEY ", term); + } } diff --git a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/InRelationBuilder.java b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/InRelationBuilder.java index d3fc8dce91d..afaa19ff724 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/InRelationBuilder.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/relation/InRelationBuilder.java @@ -50,6 +50,38 @@ default ResultT in(@NonNull Term... alternatives) { return in(Arrays.asList(alternatives)); } + /** + * Builds a NOT IN relation where the whole set of possible values is a bound variable, as in + * {@code NOT IN ?}. + * + *

Note that NOT IN support is only available in Cassandra 5.1 or later. See CASSANDRA-18584 for more + * information. + */ + @NonNull + default ResultT notIn(@NonNull BindMarker bindMarker) { + return build(" NOT IN ", bindMarker); + } + + /** + * Builds an IN relation where the arguments are the possible values, as in {@code IN (term1, + * term2...)}. + * + *

Note that NOT IN support is only available in Cassandra 5.1 or later. See CASSANDRA-18584 for more + * information. + */ + @NonNull + default ResultT notIn(@NonNull Iterable alternatives) { + return build(" NOT IN ", QueryBuilder.tuple(alternatives)); + } + + /** Var-arg equivalent of {@link #notIn(Iterable)} . */ + @NonNull + default ResultT notIn(@NonNull Term... alternatives) { + return notIn(Arrays.asList(alternatives)); + } + @NonNull ResultT build(@NonNull String operator, @Nullable Term rightOperand); } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/relation/RelationTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/relation/RelationTest.java index 515a336f5f4..ec121eaa050 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/relation/RelationTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/relation/RelationTest.java @@ -19,10 +19,12 @@ import static com.datastax.oss.driver.api.querybuilder.Assertions.assertThat; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.bindMarker; +import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.raw; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.selectFrom; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.tuple; +import org.assertj.core.util.Lists; import org.junit.Test; public class RelationTest { @@ -42,13 +44,78 @@ public void should_generate_is_not_null_relation() { } @Test - public void should_generate_in_relation() { + public void should_generate_contains_relation() { + assertThat(selectFrom("foo").all().where(Relation.column("k").contains(literal(1)))) + .hasCql("SELECT * FROM foo WHERE k CONTAINS 1"); + } + + @Test + public void should_generate_contains_key_relation() { + assertThat(selectFrom("foo").all().where(Relation.column("k").containsKey(literal(1)))) + .hasCql("SELECT * FROM foo WHERE k CONTAINS KEY 1"); + } + + @Test + public void should_generate_not_contains_relation() { + assertThat(selectFrom("foo").all().where(Relation.column("k").notContains(literal(1)))) + .hasCql("SELECT * FROM foo WHERE k NOT CONTAINS 1"); + } + + @Test + public void should_generate_not_contains_key_relation() { + assertThat(selectFrom("foo").all().where(Relation.column("k").notContainsKey(literal(1)))) + .hasCql("SELECT * FROM foo WHERE k NOT CONTAINS KEY 1"); + } + + @Test + public void should_generate_in_relation_bind_markers() { assertThat(selectFrom("foo").all().where(Relation.column("k").in(bindMarker()))) .hasCql("SELECT * FROM foo WHERE k IN ?"); assertThat(selectFrom("foo").all().where(Relation.column("k").in(bindMarker(), bindMarker()))) .hasCql("SELECT * FROM foo WHERE k IN (?,?)"); } + @Test + public void should_generate_in_relation_terms() { + assertThat( + selectFrom("foo") + .all() + .where( + Relation.column("k") + .in(Lists.newArrayList(literal(1), literal(2), literal(3))))) + .hasCql("SELECT * FROM foo WHERE k IN (1,2,3)"); + assertThat( + selectFrom("foo") + .all() + .where(Relation.column("k").in(literal(1), literal(2), literal(3)))) + .hasCql("SELECT * FROM foo WHERE k IN (1,2,3)"); + } + + @Test + public void should_generate_not_in_relation_bind_markers() { + assertThat(selectFrom("foo").all().where(Relation.column("k").notIn(bindMarker()))) + .hasCql("SELECT * FROM foo WHERE k NOT IN ?"); + assertThat( + selectFrom("foo").all().where(Relation.column("k").notIn(bindMarker(), bindMarker()))) + .hasCql("SELECT * FROM foo WHERE k NOT IN (?,?)"); + } + + @Test + public void should_generate_not_in_relation_terms() { + assertThat( + selectFrom("foo") + .all() + .where( + Relation.column("k") + .notIn(Lists.newArrayList(literal(1), literal(2), literal(3))))) + .hasCql("SELECT * FROM foo WHERE k NOT IN (1,2,3)"); + assertThat( + selectFrom("foo") + .all() + .where(Relation.column("k").notIn(literal(1), literal(2), literal(3)))) + .hasCql("SELECT * FROM foo WHERE k NOT IN (1,2,3)"); + } + @Test public void should_generate_token_relation() { assertThat(selectFrom("foo").all().where(Relation.token("k1", "k2").isEqualTo(bindMarker("t")))) From dfe11a8b671d76be6c4e90981a736325b0e4719b Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Sat, 14 Sep 2024 06:43:15 -0400 Subject: [PATCH 21/48] Fix CustomCcmRule to drop `CURRENT` flag no matter what If super.after() throws an Exception `CURRENT` flag is never dropped which leads next tests to fail with IllegalStateException("Attempting to use a Ccm rule while another is in use. This is disallowed") Patch by Dmitry Kropachev; reviewed by Andy Tolbert and Bret McGuire for JAVA-3117 --- .../oss/driver/api/testinfra/ccm/CustomCcmRule.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java index cf150b12f55..5ea1bf7ed3c 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CustomCcmRule.java @@ -53,6 +53,7 @@ protected void before() { after(); } catch (Exception e1) { LOG.warn("Error cleaning up CustomCcmRule before() failure", e1); + e.addSuppressed(e1); } throw e; } @@ -64,8 +65,11 @@ protected void before() { @Override protected void after() { - super.after(); - CURRENT.compareAndSet(this, null); + try { + super.after(); + } finally { + CURRENT.compareAndSet(this, null); + } } public CcmBridge getCcmBridge() { From 787783770b0a624f4e58d35aeff16eff8bcddc02 Mon Sep 17 00:00:00 2001 From: janehe Date: Wed, 30 Oct 2024 11:34:11 -0700 Subject: [PATCH 22/48] JAVA-3051: Memory leak patch by Jane He; reviewed by Alexandre Dutra and Bret McGuire for JAVA-3051 --- .../core/control/ControlConnection.java | 21 ++-- .../DefaultLoadBalancingPolicy.java | 96 ++++++++++++------- .../metadata/LoadBalancingPolicyWrapper.java | 12 ++- .../core/metrics/AbstractMetricUpdater.java | 5 +- .../internal/core/session/DefaultSession.java | 14 ++- .../util/concurrent/ReplayingEventFilter.java | 1 + ...faultLoadBalancingPolicyQueryPlanTest.java | 9 +- ...LoadBalancingPolicyRequestTrackerTest.java | 52 +++++----- 8 files changed, 126 insertions(+), 84 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java index 5ee9c6e7810..5c29a9b704b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java @@ -253,8 +253,8 @@ private class SingleThreaded { private final Reconnection reconnection; private DriverChannelOptions channelOptions; // The last events received for each node - private final Map lastDistanceEvents = new WeakHashMap<>(); - private final Map lastStateEvents = new WeakHashMap<>(); + private final Map lastNodeDistance = new WeakHashMap<>(); + private final Map lastNodeState = new WeakHashMap<>(); private SingleThreaded(InternalDriverContext context) { this.context = context; @@ -366,8 +366,8 @@ private void connect( .whenCompleteAsync( (channel, error) -> { try { - DistanceEvent lastDistanceEvent = lastDistanceEvents.get(node); - NodeStateEvent lastStateEvent = lastStateEvents.get(node); + NodeDistance lastDistance = lastNodeDistance.get(node); + NodeState lastState = lastNodeState.get(node); if (error != null) { if (closeWasCalled || initFuture.isCancelled()) { onSuccess.run(); // abort, we don't really care about the result @@ -406,8 +406,7 @@ private void connect( channel); channel.forceClose(); onSuccess.run(); - } else if (lastDistanceEvent != null - && lastDistanceEvent.distance == NodeDistance.IGNORED) { + } else if (lastDistance == NodeDistance.IGNORED) { LOG.debug( "[{}] New channel opened ({}) but node became ignored, " + "closing and trying next node", @@ -415,9 +414,9 @@ private void connect( channel); channel.forceClose(); connect(nodes, errors, onSuccess, onFailure); - } else if (lastStateEvent != null - && (lastStateEvent.newState == null /*(removed)*/ - || lastStateEvent.newState == NodeState.FORCED_DOWN)) { + } else if (lastNodeState.containsKey(node) + && (lastState == null /*(removed)*/ + || lastState == NodeState.FORCED_DOWN)) { LOG.debug( "[{}] New channel opened ({}) but node was removed or forced down, " + "closing and trying next node", @@ -534,7 +533,7 @@ private void reconnectNow() { private void onDistanceEvent(DistanceEvent event) { assert adminExecutor.inEventLoop(); - this.lastDistanceEvents.put(event.node, event); + this.lastNodeDistance.put(event.node, event.distance); if (event.distance == NodeDistance.IGNORED && channel != null && !channel.closeFuture().isDone() @@ -549,7 +548,7 @@ private void onDistanceEvent(DistanceEvent event) { private void onStateEvent(NodeStateEvent event) { assert adminExecutor.inEventLoop(); - this.lastStateEvents.put(event.node, event); + this.lastNodeState.put(event.node, event.newState); if ((event.newState == null /*(removed)*/ || event.newState == NodeState.FORCED_DOWN) && channel != null && !channel.closeFuture().isDone() diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java index 47edcdfe53e..0f03cbb3643 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicy.java @@ -33,15 +33,19 @@ import com.datastax.oss.driver.internal.core.util.ArrayUtils; import com.datastax.oss.driver.internal.core.util.collection.QueryPlan; import com.datastax.oss.driver.internal.core.util.collection.SimpleQueryPlan; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; +import com.datastax.oss.driver.shaded.guava.common.collect.MapMaker; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.util.BitSet; import java.util.Map; import java.util.Optional; +import java.util.OptionalLong; import java.util.Queue; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicLongArray; import net.jcip.annotations.ThreadSafe; @@ -96,7 +100,7 @@ public class DefaultLoadBalancingPolicy extends BasicLoadBalancingPolicy impleme private static final int MAX_IN_FLIGHT_THRESHOLD = 10; private static final long RESPONSE_COUNT_RESET_INTERVAL_NANOS = MILLISECONDS.toNanos(200); - protected final Map responseTimes = new ConcurrentHashMap<>(); + protected final ConcurrentMap responseTimes; protected final Map upTimes = new ConcurrentHashMap<>(); private final boolean avoidSlowReplicas; @@ -104,6 +108,7 @@ public DefaultLoadBalancingPolicy(@NonNull DriverContext context, @NonNull Strin super(context, profileName); this.avoidSlowReplicas = profile.getBoolean(DefaultDriverOption.LOAD_BALANCING_POLICY_SLOW_AVOIDANCE, true); + this.responseTimes = new MapMaker().weakKeys().makeMap(); } @NonNull @@ -274,40 +279,19 @@ protected boolean isBusy(@NonNull Node node, @NonNull Session session) { } protected boolean isResponseRateInsufficient(@NonNull Node node, long now) { - // response rate is considered insufficient when less than 2 responses were obtained in - // the past interval delimited by RESPONSE_COUNT_RESET_INTERVAL_NANOS. - if (responseTimes.containsKey(node)) { - AtomicLongArray array = responseTimes.get(node); - if (array.length() == 2) { - long threshold = now - RESPONSE_COUNT_RESET_INTERVAL_NANOS; - long leastRecent = array.get(0); - return leastRecent - threshold < 0; - } - } - return true; + NodeResponseRateSample sample = responseTimes.get(node); + return !(sample == null || sample.hasSufficientResponses(now)); } + /** + * Synchronously updates the response times for the given node. It is synchronous because the + * {@link #DefaultLoadBalancingPolicy(com.datastax.oss.driver.api.core.context.DriverContext, + * java.lang.String) CacheLoader.load} assigned is synchronous. + * + * @param node The node to update. + */ protected void updateResponseTimes(@NonNull Node node) { - responseTimes.compute( - node, - (n, array) -> { - // The array stores at most two timestamps, since we don't need more; - // the first one is always the least recent one, and hence the one to inspect. - long now = nanoTime(); - if (array == null) { - array = new AtomicLongArray(1); - array.set(0, now); - } else if (array.length() == 1) { - long previous = array.get(0); - array = new AtomicLongArray(2); - array.set(0, previous); - array.set(1, now); - } else { - array.set(0, array.get(1)); - array.set(1, now); - } - return array; - }); + this.responseTimes.compute(node, (k, v) -> v == null ? new NodeResponseRateSample() : v.next()); } protected int getInFlight(@NonNull Node node, @NonNull Session session) { @@ -318,4 +302,52 @@ protected int getInFlight(@NonNull Node node, @NonNull Session session) { // processing them). return (pool == null) ? 0 : pool.getInFlight(); } + + protected class NodeResponseRateSample { + + @VisibleForTesting protected final long oldest; + @VisibleForTesting protected final OptionalLong newest; + + private NodeResponseRateSample() { + long now = nanoTime(); + this.oldest = now; + this.newest = OptionalLong.empty(); + } + + private NodeResponseRateSample(long oldestSample) { + this(oldestSample, nanoTime()); + } + + private NodeResponseRateSample(long oldestSample, long newestSample) { + this.oldest = oldestSample; + this.newest = OptionalLong.of(newestSample); + } + + @VisibleForTesting + protected NodeResponseRateSample(AtomicLongArray times) { + assert times.length() >= 1; + this.oldest = times.get(0); + this.newest = (times.length() > 1) ? OptionalLong.of(times.get(1)) : OptionalLong.empty(); + } + + // Our newest sample becomes the oldest in the next generation + private NodeResponseRateSample next() { + return new NodeResponseRateSample(this.getNewestValidSample(), nanoTime()); + } + + // If we have a pair of values return the newest, otherwise we have just one value... so just + // return it + private long getNewestValidSample() { + return this.newest.orElse(this.oldest); + } + + // response rate is considered insufficient when less than 2 responses were obtained in + // the past interval delimited by RESPONSE_COUNT_RESET_INTERVAL_NANOS. + private boolean hasSufficientResponses(long now) { + // If we only have one sample it's an automatic failure + if (!this.newest.isPresent()) return true; + long threshold = now - RESPONSE_COUNT_RESET_INTERVAL_NANOS; + return this.oldest - threshold >= 0; + } + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java index 20d045d4e72..5c8473a3b67 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/LoadBalancingPolicyWrapper.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import java.util.WeakHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; @@ -105,7 +106,7 @@ public LoadBalancingPolicyWrapper( // Just an alias to make the rest of the code more readable this.policies = reporters.keySet(); - this.distances = new HashMap<>(); + this.distances = new WeakHashMap<>(); this.logPrefix = context.getSessionName(); context.getEventBus().register(NodeStateEvent.class, this::onNodeStateEvent); @@ -172,6 +173,7 @@ private void onNodeStateEvent(NodeStateEvent event) { // once it has gone through the filter private void processNodeStateEvent(NodeStateEvent event) { + DefaultNode node = event.node; switch (stateRef.get()) { case BEFORE_INIT: case DURING_INIT: @@ -181,13 +183,13 @@ private void processNodeStateEvent(NodeStateEvent event) { case RUNNING: for (LoadBalancingPolicy policy : policies) { if (event.newState == NodeState.UP) { - policy.onUp(event.node); + policy.onUp(node); } else if (event.newState == NodeState.DOWN || event.newState == NodeState.FORCED_DOWN) { - policy.onDown(event.node); + policy.onDown(node); } else if (event.newState == NodeState.UNKNOWN) { - policy.onAdd(event.node); + policy.onAdd(node); } else if (event.newState == null) { - policy.onRemove(event.node); + policy.onRemove(node); } else { LOG.warn("[{}] Unsupported event: {}", logPrefix, event); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java index 5e2392a2e7f..3d7dc50a7c0 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metrics/AbstractMetricUpdater.java @@ -173,9 +173,8 @@ protected Timeout newTimeout() { .getTimer() .newTimeout( t -> { - if (t.isExpired()) { - clearMetrics(); - } + clearMetrics(); + cancelMetricsExpirationTimeout(); }, expireAfter.toNanos(), TimeUnit.NANOSECONDS); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java index 6f063ae9a50..b795c30fce7 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java @@ -527,14 +527,18 @@ private void notifyListeners() { private void onNodeStateChanged(NodeStateEvent event) { assert adminExecutor.inEventLoop(); - if (event.newState == null) { - context.getNodeStateListener().onRemove(event.node); + DefaultNode node = event.node; + if (node == null) { + LOG.debug( + "[{}] Node for this event was removed, ignoring state change: {}", logPrefix, event); + } else if (event.newState == null) { + context.getNodeStateListener().onRemove(node); } else if (event.oldState == null && event.newState == NodeState.UNKNOWN) { - context.getNodeStateListener().onAdd(event.node); + context.getNodeStateListener().onAdd(node); } else if (event.newState == NodeState.UP) { - context.getNodeStateListener().onUp(event.node); + context.getNodeStateListener().onUp(node); } else if (event.newState == NodeState.DOWN || event.newState == NodeState.FORCED_DOWN) { - context.getNodeStateListener().onDown(event.node); + context.getNodeStateListener().onDown(node); } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/ReplayingEventFilter.java b/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/ReplayingEventFilter.java index 12679db7ff0..27ca1b6ff42 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/ReplayingEventFilter.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/util/concurrent/ReplayingEventFilter.java @@ -82,6 +82,7 @@ public void markReady() { consumer.accept(event); } } finally { + recordedEvents.clear(); stateLock.writeLock().unlock(); } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyQueryPlanTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyQueryPlanTest.java index 6098653bc2e..fff86a1b750 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyQueryPlanTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyQueryPlanTest.java @@ -203,7 +203,10 @@ public void should_prioritize_and_shuffle_3_or_more_replicas_when_first_unhealth given(pool3.getInFlight()).willReturn(0); given(pool5.getInFlight()).willReturn(0); - dsePolicy.responseTimes.put(node1, new AtomicLongArray(new long[] {T0, T0})); // unhealthy + dsePolicy.responseTimes.put( + node1, + dsePolicy + .new NodeResponseRateSample(new AtomicLongArray(new long[] {T0, T0}))); // unhealthy // When Queue plan1 = dsePolicy.newQueryPlan(request, session); @@ -232,7 +235,9 @@ public void should_prioritize_and_shuffle_3_or_more_replicas_when_first_unhealth given(pool3.getInFlight()).willReturn(0); given(pool5.getInFlight()).willReturn(0); - dsePolicy.responseTimes.put(node1, new AtomicLongArray(new long[] {T1, T1})); // healthy + dsePolicy.responseTimes.put( + node1, + dsePolicy.new NodeResponseRateSample(new AtomicLongArray(new long[] {T1, T1}))); // healthy // When Queue plan1 = dsePolicy.newQueryPlan(request, session); diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyRequestTrackerTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyRequestTrackerTest.java index bcc6439a2a5..757af43ef67 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyRequestTrackerTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/loadbalancing/DefaultLoadBalancingPolicyRequestTrackerTest.java @@ -69,11 +69,11 @@ public void should_record_first_response_time_on_node_success() { // Then assertThat(policy.responseTimes) - .hasEntrySatisfying(node1, value -> assertThat(value.get(0)).isEqualTo(123L)) + .hasEntrySatisfying(node1, value -> assertThat(value.oldest).isEqualTo(123L)) .doesNotContainKeys(node2, node3); - assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -91,13 +91,13 @@ public void should_record_second_response_time_on_node_success() { node1, value -> { // oldest value first - assertThat(value.get(0)).isEqualTo(123); - assertThat(value.get(1)).isEqualTo(456); + assertThat(value.oldest).isEqualTo(123); + assertThat(value.newest.getAsLong()).isEqualTo(456); }) .doesNotContainKeys(node2, node3); assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -116,14 +116,14 @@ public void should_record_further_response_times_on_node_success() { node1, value -> { // values should rotate left (bubble up) - assertThat(value.get(0)).isEqualTo(456); - assertThat(value.get(1)).isEqualTo(789); + assertThat(value.oldest).isEqualTo(456); + assertThat(value.newest.getAsLong()).isEqualTo(789); }) - .hasEntrySatisfying(node2, value -> assertThat(value.get(0)).isEqualTo(789)) + .hasEntrySatisfying(node2, value -> assertThat(value.oldest).isEqualTo(789)) .doesNotContainKey(node3); assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -137,11 +137,11 @@ public void should_record_first_response_time_on_node_error() { // Then assertThat(policy.responseTimes) - .hasEntrySatisfying(node1, value -> assertThat(value.get(0)).isEqualTo(123L)) + .hasEntrySatisfying(node1, value -> assertThat(value.oldest).isEqualTo(123L)) .doesNotContainKeys(node2, node3); - assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -160,13 +160,13 @@ public void should_record_second_response_time_on_node_error() { node1, value -> { // oldest value first - assertThat(value.get(0)).isEqualTo(123); - assertThat(value.get(1)).isEqualTo(456); + assertThat(value.oldest).isEqualTo(123); + assertThat(value.newest.getAsLong()).isEqualTo(456); }) .doesNotContainKeys(node2, node3); assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } @Test @@ -186,13 +186,13 @@ public void should_record_further_response_times_on_node_error() { node1, value -> { // values should rotate left (bubble up) - assertThat(value.get(0)).isEqualTo(456); - assertThat(value.get(1)).isEqualTo(789); + assertThat(value.oldest).isEqualTo(456); + assertThat(value.newest.getAsLong()).isEqualTo(789); }) - .hasEntrySatisfying(node2, value -> assertThat(value.get(0)).isEqualTo(789)) + .hasEntrySatisfying(node2, value -> assertThat(value.oldest).isEqualTo(789)) .doesNotContainKey(node3); assertThat(policy.isResponseRateInsufficient(node1, nextNanoTime)).isFalse(); - assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isTrue(); - assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isTrue(); + assertThat(policy.isResponseRateInsufficient(node2, nextNanoTime)).isFalse(); + assertThat(policy.isResponseRateInsufficient(node3, nextNanoTime)).isFalse(); } } From a4175f33e43344fd1b092aae332ed5979a1bd831 Mon Sep 17 00:00:00 2001 From: "Siyao (Jane) He" Date: Mon, 28 Oct 2024 14:44:22 -0700 Subject: [PATCH 23/48] Automate latest Cassandra versions when running CI patch by Siyao (Jane) He; reviewed by Mick Semb Wever for CASSJAVA-25 --- Jenkinsfile-asf | 7 ++++--- ci/run-tests.sh | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile-asf b/Jenkinsfile-asf index a1be4bcd4f6..0217d0455d6 100644 --- a/Jenkinsfile-asf +++ b/Jenkinsfile-asf @@ -39,9 +39,10 @@ pipeline { } axis { name 'SERVER_VERSION' - values '3.11.17', - '4.0.13', - '5.0-beta1' + values '3.11', + '4.0', + '4.1', + '5.0' } } stages { diff --git a/ci/run-tests.sh b/ci/run-tests.sh index 02a8070e7a9..5268bdd7113 100755 --- a/ci/run-tests.sh +++ b/ci/run-tests.sh @@ -6,5 +6,7 @@ cd $(dirname "$(readlink -f "$0")")/.. printenv | sort mvn -B -V install -DskipTests -Dmaven.javadoc.skip=true jabba use ${TEST_JAVA_VERSION} +# Find out the latest patch version of Cassandra +PATCH_SERVER_VERSION=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP '(?<=href=\")[0-9]+\.[0-9]+\.[0-9]+(?=)' | sort -rV | uniq -w 3 | grep $SERVER_VERSION) printenv | sort -mvn -B -V verify -T 1 -Ptest-jdk-${TEST_JAVA_MAJOR_VERSION} -DtestJavaHome=$(jabba which ${TEST_JAVA_VERSION}) -Dccm.version=${SERVER_VERSION} -Dccm.dse=false -Dmaven.test.failure.ignore=true -Dmaven.javadoc.skip=true +mvn -B -V verify -T 1 -Ptest-jdk-${TEST_JAVA_MAJOR_VERSION} -DtestJavaHome=$(jabba which ${TEST_JAVA_VERSION}) -Dccm.version=${PATCH_SERVER_VERSION} -Dccm.dse=false -Dmaven.test.failure.ignore=true -Dmaven.javadoc.skip=true From 01c6151cd4a3c1dbbd4d1251fb453305385668e1 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Sun, 8 Sep 2024 15:06:53 +0200 Subject: [PATCH 24/48] Refactor integration tests to support multiple C* distributions. Test with DataStax HCD 1.0.0 patch by Lukasz Antoniak; reviewed by Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1958 --- Jenkinsfile-datastax | 34 +++- .../datastax/oss/driver/api/core/Version.java | 1 + .../cql/continuous/ContinuousPagingIT.java | 8 +- .../remote/GraphTraversalRemoteITBase.java | 5 +- .../graph/statement/GraphTraversalITBase.java | 5 +- .../schema/DseAggregateMetadataIT.java | 6 +- .../schema/DseFunctionMetadataIT.java | 6 +- .../core/compression/DirectCompressionIT.java | 6 +- .../core/compression/HeapCompressionIT.java | 3 +- .../oss/driver/core/cql/QueryTraceIT.java | 3 +- .../oss/driver/core/metadata/DescribeIT.java | 28 +-- .../driver/core/metadata/NodeMetadataIT.java | 7 +- .../driver/core/metadata/SchemaChangesIT.java | 5 +- .../oss/driver/core/metadata/SchemaIT.java | 8 +- .../oss/driver/mapper/DeleteReactiveIT.java | 5 +- .../oss/driver/mapper/InventoryITBase.java | 11 +- .../src/test/resources/DescribeIT/hcd/1.0.cql | 186 ++++++++++++++++++ osgi-tests/README.md | 4 +- .../osgi/support/CcmStagedReactor.java | 8 +- test-infra/revapi.json | 21 ++ .../driver/api/testinfra/ccm/BaseCcmRule.java | 24 ++- .../driver/api/testinfra/ccm/CcmBridge.java | 83 ++++---- .../DefaultCcmBridgeBuilderCustomizer.java | 8 +- .../ccm/DistributionCassandraVersions.java | 57 ++++++ .../requirement/BackendRequirementRule.java | 2 +- .../testinfra/requirement/BackendType.java | 13 +- .../api/testinfra/session/SessionRule.java | 13 +- 27 files changed, 439 insertions(+), 121 deletions(-) create mode 100644 integration-tests/src/test/resources/DescribeIT/hcd/1.0.cql create mode 100644 test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DistributionCassandraVersions.java diff --git a/Jenkinsfile-datastax b/Jenkinsfile-datastax index 4cc20d79604..af3faafee20 100644 --- a/Jenkinsfile-datastax +++ b/Jenkinsfile-datastax @@ -61,7 +61,7 @@ def initializeEnvironment() { . ${JABBA_SHELL} jabba which 1.8''', returnStdout: true).trim() - sh label: 'Download Apache CassandraⓇ or DataStax Enterprise',script: '''#!/bin/bash -le + sh label: 'Download Apache CassandraⓇ, DataStax Enterprise or DataStax HCD ',script: '''#!/bin/bash -le . ${JABBA_SHELL} jabba use 1.8 . ${CCM_ENVIRONMENT_SHELL} ${SERVER_VERSION} @@ -75,13 +75,26 @@ CCM_CASSANDRA_VERSION=${DSE_FIXED_VERSION} # maintain for backwards compatibilit CCM_VERSION=${DSE_FIXED_VERSION} CCM_SERVER_TYPE=dse DSE_VERSION=${DSE_FIXED_VERSION} -CCM_IS_DSE=true CCM_BRANCH=${DSE_FIXED_VERSION} DSE_BRANCH=${DSE_FIXED_VERSION} ENVIRONMENT_EOF ''' } + if (env.SERVER_VERSION.split('-')[0] == 'hcd') { + env.HCD_FIXED_VERSION = env.SERVER_VERSION.split('-')[1] + sh label: 'Update environment for DataStax HCD', script: '''#!/bin/bash -le + cat >> ${HOME}/environment.txt << ENVIRONMENT_EOF +CCM_CASSANDRA_VERSION=${HCD_FIXED_VERSION} # maintain for backwards compatibility +CCM_VERSION=${HCD_FIXED_VERSION} +CCM_SERVER_TYPE=hcd +HCD_VERSION=${HCD_FIXED_VERSION} +CCM_BRANCH=${HCD_FIXED_VERSION} +HCD_BRANCH=${HCD_FIXED_VERSION} +ENVIRONMENT_EOF + ''' + } + sh label: 'Display Java and environment information',script: '''#!/bin/bash -le # Load CCM environment variables set -o allexport @@ -144,7 +157,7 @@ def executeTests() { -Dmaven.test.failure.ignore=true \ -Dmaven.javadoc.skip=${SKIP_JAVADOCS} \ -Dccm.version=${CCM_CASSANDRA_VERSION} \ - -Dccm.dse=${CCM_IS_DSE} \ + -Dccm.distribution=${CCM_SERVER_TYPE:cassandra} \ -Dproxy.path=${HOME}/proxy \ ${SERIAL_ITS_ARGUMENT} \ ${ISOLATED_ITS_ARGUMENT} \ @@ -269,6 +282,7 @@ pipeline { 'dse-6.7.17', // Previous DataStax Enterprise 'dse-6.8.30', // Current DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise + 'hcd-1.0.0', // Current DataStax HCD 'ALL'], description: '''Apache Cassandra® and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS builds @@ -330,6 +344,10 @@ pipeline { + + + +
dse-6.9.0 DataStax Enterprise v6.9.x
hcd-1.0.0DataStax HCD v1.0.x
''') choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_JABBA_VERSION', @@ -421,9 +439,9 @@ pipeline { H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 4.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 # Every weeknight (Monday - Friday) around 12:00 PM noon ### JDK11 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 ### JDK17 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 """ : "") } @@ -460,7 +478,8 @@ pipeline { values '3.11', // Latest stable Apache CassandraⓇ '4.1', // Development Apache CassandraⓇ 'dse-6.8.30', // Current DataStax Enterprise - 'dse-6.9.0' // Current DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise + 'hcd-1.0.0' // Current DataStax HCD } axis { name 'JABBA_VERSION' @@ -578,7 +597,8 @@ pipeline { 'dse-6.0.18', // Previous DataStax Enterprise 'dse-6.7.17', // Previous DataStax Enterprise 'dse-6.8.30', // Current DataStax Enterprise - 'dse-6.9.0' // Current DataStax Enterprise + 'dse-6.9.0', // Current DataStax Enterprise + 'hcd-1.0.0' // Current DataStax HCD } } when { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/Version.java b/core/src/main/java/com/datastax/oss/driver/api/core/Version.java index 4de006da268..52751e02984 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/Version.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/Version.java @@ -48,6 +48,7 @@ public class Version implements Comparable, Serializable { private static final Pattern pattern = Pattern.compile(VERSION_REGEXP); + @NonNull public static final Version V1_0_0 = Objects.requireNonNull(parse("1.0.0")); @NonNull public static final Version V2_1_0 = Objects.requireNonNull(parse("2.1.0")); @NonNull public static final Version V2_2_0 = Objects.requireNonNull(parse("2.2.0")); @NonNull public static final Version V3_0_0 = Objects.requireNonNull(parse("3.0.0")); diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/cql/continuous/ContinuousPagingIT.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/cql/continuous/ContinuousPagingIT.java index 24ee5c0373d..45cc84f0719 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/cql/continuous/ContinuousPagingIT.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/cql/continuous/ContinuousPagingIT.java @@ -46,7 +46,6 @@ import java.time.Duration; import java.util.Collections; import java.util.Iterator; -import java.util.Objects; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; @@ -281,11 +280,8 @@ public void prepared_statement_paging_should_be_resilient_to_schema_change() { // dropped. Row row = it.next(); assertThat(row.getString("k")).isNotNull(); - if (ccmRule - .getDseVersion() - .orElseThrow(IllegalStateException::new) - .compareTo(Objects.requireNonNull(Version.parse("6.0.0"))) - >= 0) { + if (ccmRule.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(Version.parse("6.0.0")) >= 0)) { // DSE 6 only, v should be null here since dropped. // Not reliable for 5.1 since we may have gotten page queued before schema changed. assertThat(row.isNull("v")).isTrue(); diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/remote/GraphTraversalRemoteITBase.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/remote/GraphTraversalRemoteITBase.java index 69949951378..3db8a7d1a12 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/remote/GraphTraversalRemoteITBase.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/remote/GraphTraversalRemoteITBase.java @@ -30,6 +30,7 @@ import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.Version; import com.datastax.oss.driver.api.core.servererrors.InvalidQueryException; +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirement; import com.datastax.oss.driver.api.testinfra.requirement.BackendType; @@ -643,9 +644,9 @@ public void should_allow_use_of_dsl_graph_binary() { */ @Test public void should_return_correct_results_when_bulked() { - Optional dseVersion = ccmRule().getCcmBridge().getDseVersion(); Assumptions.assumeThat( - dseVersion.isPresent() && dseVersion.get().compareTo(Version.parse("5.1.2")) > 0) + CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(Version.parse("5.1.2")) > 0)) .isTrue(); List results = graphTraversalSource().E().label().barrier().toList(); diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/statement/GraphTraversalITBase.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/statement/GraphTraversalITBase.java index 98d9ccf1b80..5bcb01bc165 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/statement/GraphTraversalITBase.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/graph/statement/GraphTraversalITBase.java @@ -36,7 +36,9 @@ import com.datastax.oss.driver.api.core.Version; import com.datastax.oss.driver.api.core.servererrors.InvalidQueryException; import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import java.util.List; @@ -598,7 +600,8 @@ public void should_allow_use_of_dsl_graph_binary() throws Exception { @Test public void should_return_correct_results_when_bulked() { Assumptions.assumeThat( - ccmRule().getCcmBridge().getDseVersion().get().compareTo(Version.parse("5.1.2")) > 0) + CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(Version.parse("5.1.2")) > 0)) .isTrue(); GraphResultSet rs = diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseAggregateMetadataIT.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseAggregateMetadataIT.java index b0e989e86a3..4c899fa5e63 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseAggregateMetadataIT.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseAggregateMetadataIT.java @@ -106,9 +106,9 @@ public void should_parse_aggregate_with_deterministic() { } private static boolean isDse6OrHigher() { - assumeThat(CCM_RULE.getDseVersion()) + assumeThat(CCM_RULE.isDistributionOf(BackendType.DSE)) .describedAs("DSE required for DseFunctionMetadata tests") - .isPresent(); - return CCM_RULE.getDseVersion().get().compareTo(DSE_6_0_0) >= 0; + .isTrue(); + return CCM_RULE.getDistributionVersion().compareTo(DSE_6_0_0) >= 0; } } diff --git a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseFunctionMetadataIT.java b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseFunctionMetadataIT.java index 53e2d1be8f8..53559a66b1b 100644 --- a/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseFunctionMetadataIT.java +++ b/integration-tests/src/test/java/com/datastax/dse/driver/api/core/metadata/schema/DseFunctionMetadataIT.java @@ -233,9 +233,9 @@ public void should_parse_function_with_deterministic_and_monotonic_on() { } private static boolean isDse6OrHigher() { - assumeThat(CCM_RULE.getDseVersion()) + assumeThat(CCM_RULE.isDistributionOf(BackendType.DSE)) .describedAs("DSE required for DseFunctionMetadata tests") - .isPresent(); - return CCM_RULE.getDseVersion().get().compareTo(DSE_6_0_0) >= 0; + .isTrue(); + return CCM_RULE.getDistributionVersion().compareTo(DSE_6_0_0) >= 0; } } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/DirectCompressionIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/DirectCompressionIT.java index 51f71f85b5c..3dad08f4de6 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/DirectCompressionIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/DirectCompressionIT.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.ParallelizableTests; @@ -75,8 +76,9 @@ public static void setup() { public void should_execute_queries_with_snappy_compression() throws Exception { Assume.assumeTrue( "Snappy is not supported in OSS C* 4.0+ with protocol v5", - CCM_RULE.getDseVersion().isPresent() - || CCM_RULE.getCassandraVersion().nextStable().compareTo(Version.V4_0_0) < 0); + !CCM_RULE.isDistributionOf(BackendType.HCD) + && (CCM_RULE.isDistributionOf(BackendType.DSE) + || CCM_RULE.getCassandraVersion().nextStable().compareTo(Version.V4_0_0) < 0)); createAndCheckCluster("snappy"); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/HeapCompressionIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/HeapCompressionIT.java index 466a9d87ac3..a14c3b29b21 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/HeapCompressionIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/compression/HeapCompressionIT.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.IsolatedTests; @@ -79,7 +80,7 @@ public static void setup() { public void should_execute_queries_with_snappy_compression() throws Exception { Assume.assumeTrue( "Snappy is not supported in OSS C* 4.0+ with protocol v5", - CCM_RULE.getDseVersion().isPresent() + CCM_RULE.isDistributionOf(BackendType.DSE) || CCM_RULE.getCassandraVersion().nextStable().compareTo(Version.V4_0_0) < 0); createAndCheckCluster("snappy"); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/QueryTraceIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/QueryTraceIT.java index f4ac85d6629..37a600efbc4 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/QueryTraceIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/QueryTraceIT.java @@ -28,6 +28,7 @@ import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.metadata.EndPoint; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.categories.ParallelizableTests; import java.net.InetAddress; @@ -82,7 +83,7 @@ public void should_fetch_trace_when_tracing_enabled() { InetAddress nodeAddress = ((InetSocketAddress) contactPoint.resolve()).getAddress(); boolean expectPorts = CCM_RULE.getCassandraVersion().nextStable().compareTo(Version.V4_0_0) >= 0 - && !CCM_RULE.getDseVersion().isPresent(); + && !CCM_RULE.isDistributionOf(BackendType.DSE); QueryTrace queryTrace = executionInfo.getQueryTrace(); assertThat(queryTrace.getTracingId()).isEqualTo(executionInfo.getTracingId()); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java index 9fbf5e355eb..4d6c2a7a3b1 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/DescribeIT.java @@ -29,6 +29,7 @@ import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.datastax.oss.driver.categories.ParallelizableTests; @@ -37,12 +38,14 @@ import com.datastax.oss.driver.internal.core.metadata.schema.DefaultTableMetadata; import com.datastax.oss.driver.shaded.guava.common.base.Charsets; import com.datastax.oss.driver.shaded.guava.common.base.Splitter; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import com.google.common.io.Files; import java.io.File; import java.io.IOException; import java.net.URL; import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; import org.junit.BeforeClass; @@ -79,17 +82,23 @@ public class DescribeIT { Splitter.on(Pattern.compile(";\n")).omitEmptyStrings(); private static Version serverVersion; - private static boolean isDse; + + private static final Map scriptFileForBackend = + ImmutableMap.builder() + .put(BackendType.CASSANDRA, "DescribeIT/oss") + .put(BackendType.DSE, "DescribeIT/dse") + .put(BackendType.HCD, "DescribeIT/hcd") + .build(); private static File scriptFile; private static String scriptContents; @BeforeClass public static void setup() { - Optional dseVersion = CCM_RULE.getDseVersion(); - isDse = dseVersion.isPresent(); serverVersion = - isDse ? dseVersion.get().nextStable() : CCM_RULE.getCassandraVersion().nextStable(); + CCM_RULE.isDistributionOf(BackendType.CASSANDRA) + ? CCM_RULE.getCassandraVersion().nextStable() + : CCM_RULE.getDistributionVersion().nextStable(); scriptFile = getScriptFile(); assertThat(scriptFile).exists(); @@ -114,12 +123,12 @@ public void describe_output_should_match_creation_script() throws Exception { "Describe output doesn't match create statements, " + "maybe you need to add a new script in integration-tests/src/test/resources. " + "Server version = %s %s, used script = %s", - isDse ? "DSE" : "Cassandra", serverVersion, scriptFile) + CCM_RULE.getDistribution(), serverVersion, scriptFile) .isEqualTo(scriptContents); } private boolean atLeastVersion(Version dseVersion, Version ossVersion) { - Version comparison = isDse ? dseVersion : ossVersion; + Version comparison = CCM_RULE.isDistributionOf(BackendType.DSE) ? dseVersion : ossVersion; return serverVersion.compareTo(comparison) >= 0; } @@ -138,11 +147,9 @@ public void keyspace_metadata_should_be_serializable() throws Exception { assertThat(ks.getUserDefinedTypes()).isNotEmpty(); assertThat(ks.getTables()).isNotEmpty(); if (atLeastVersion(Version.V5_0_0, Version.V3_0_0)) { - assertThat(ks.getViews()).isNotEmpty(); } if (atLeastVersion(Version.V5_0_0, Version.V2_2_0)) { - assertThat(ks.getFunctions()).isNotEmpty(); assertThat(ks.getAggregates()).isNotEmpty(); } @@ -177,7 +184,7 @@ private static File getScriptFile() { logbackTestUrl); } File resourcesDir = new File(logbackTestUrl.getFile()).getParentFile(); - File scriptsDir = new File(resourcesDir, isDse ? "DescribeIT/dse" : "DescribeIT/oss"); + File scriptsDir = new File(resourcesDir, scriptFileForBackend.get(CCM_RULE.getDistribution())); LOG.debug("Looking for a matching script in directory {}", scriptsDir); File[] candidates = scriptsDir.listFiles(); @@ -204,8 +211,7 @@ private static File getScriptFile() { .as("Could not find create script with version <= %s in %s", serverVersion, scriptsDir) .isNotNull(); - LOG.info( - "Using {} to test against {} {}", bestFile, isDse ? "DSE" : "Cassandra", serverVersion); + LOG.info("Using {} to test against {} {}", bestFile, CCM_RULE.getDistribution(), serverVersion); return bestFile; } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/NodeMetadataIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/NodeMetadataIT.java index c7b51c040b5..8f5680ff41a 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/NodeMetadataIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/NodeMetadataIT.java @@ -62,8 +62,9 @@ public void should_expose_node_metadata() { assertThat(node.getListenAddress().get().getAddress()).isEqualTo(connectAddress.getAddress()); assertThat(node.getDatacenter()).isEqualTo("dc1"); assertThat(node.getRack()).isEqualTo("r1"); - if (!CcmBridge.DSE_ENABLEMENT) { - // CcmBridge does not report accurate C* versions for DSE, only approximated values + if (CcmBridge.isDistributionOf(BackendType.CASSANDRA)) { + // CcmBridge does not report accurate C* versions for other distributions (e.g. DSE), only + // approximated values assertThat(node.getCassandraVersion()).isEqualTo(ccmRule.getCassandraVersion()); } assertThat(node.getState()).isSameAs(NodeState.UP); @@ -106,7 +107,7 @@ public void should_expose_dse_node_properties() { DseNodeProperties.DSE_WORKLOADS, DseNodeProperties.SERVER_ID); assertThat(node.getExtras().get(DseNodeProperties.DSE_VERSION)) - .isEqualTo(ccmRule.getDseVersion().get()); + .isEqualTo(ccmRule.getDistributionVersion()); assertThat(node.getExtras().get(DseNodeProperties.SERVER_ID)).isInstanceOf(String.class); assertThat(node.getExtras().get(DseNodeProperties.DSE_WORKLOADS)).isInstanceOf(Set.class); } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaChangesIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaChangesIT.java index 6f1dcb791c6..85fcfc02cdb 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaChangesIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaChangesIT.java @@ -33,6 +33,7 @@ import com.datastax.oss.driver.api.core.type.DataTypes; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import com.datastax.oss.driver.api.testinfra.session.SessionUtils; import com.google.common.collect.ImmutableList; @@ -54,8 +55,8 @@ public class SchemaChangesIT { static { CustomCcmRule.Builder builder = CustomCcmRule.builder(); - if (!CcmBridge.DSE_ENABLEMENT - && CcmBridge.VERSION.nextStable().compareTo(Version.V4_0_0) >= 0) { + if (!CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> cass.nextStable().compareTo(Version.V4_0_0) >= 0)) { builder.withCassandraConfiguration("enable_materialized_views", true); } CCM_RULE = builder.build(); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java index 805b2d970cc..df5571974c1 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/metadata/SchemaIT.java @@ -335,11 +335,9 @@ public void should_exclude_virtual_keyspaces_from_token_map() { private void skipIfDse60() { // Special case: DSE 6.0 reports C* 4.0 but does not support virtual tables - if (ccmRule.getDseVersion().isPresent()) { - Version dseVersion = ccmRule.getDseVersion().get(); - if (dseVersion.compareTo(DSE_MIN_VIRTUAL_TABLES) < 0) { - throw new AssumptionViolatedException("DSE 6.0 does not support virtual tables"); - } + if (!ccmRule.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(DSE_MIN_VIRTUAL_TABLES) >= 0)) { + throw new AssumptionViolatedException("DSE 6.0 does not support virtual tables"); } } } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java index 3a418c73653..2eb898021ba 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/DeleteReactiveIT.java @@ -37,6 +37,7 @@ import com.datastax.oss.driver.api.mapper.entity.saving.NullSavingStrategy; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.session.SessionRule; import io.reactivex.Flowable; import java.util.UUID; @@ -57,8 +58,8 @@ public class DeleteReactiveIT extends InventoryITBase { @ClassRule public static TestRule chain = RuleChain.outerRule(ccmRule).around(sessionRule); private static CustomCcmRule.Builder configureCcm(CustomCcmRule.Builder builder) { - if (!CcmBridge.DSE_ENABLEMENT - && CcmBridge.VERSION.nextStable().compareTo(Version.V4_0_0) >= 0) { + if (!CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> cass.nextStable().compareTo(Version.V4_0_0) >= 0)) { builder.withCassandraConfiguration("enable_sasi_indexes", true); } return builder; diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java index 9495003ae49..1bd899e4541 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/mapper/InventoryITBase.java @@ -23,10 +23,10 @@ import com.datastax.oss.driver.api.mapper.annotations.Entity; import com.datastax.oss.driver.api.mapper.annotations.PartitionKey; import com.datastax.oss.driver.api.testinfra.ccm.BaseCcmRule; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.UUID; /** Factors common code for mapper tests that rely on a simple inventory model. */ @@ -93,13 +93,14 @@ protected static List createStatements(BaseCcmRule ccmRule, boolean requ return builder.build(); } - private static final Version MINIMUM_SASI_VERSION = Version.parse("3.4.0"); - private static final Version BROKEN_SASI_VERSION = Version.parse("6.8.0"); + private static final Version MINIMUM_SASI_VERSION = + Objects.requireNonNull(Version.parse("3.4.0")); + private static final Version BROKEN_SASI_VERSION = Objects.requireNonNull(Version.parse("6.8.0")); protected static boolean isSasiBroken(BaseCcmRule ccmRule) { - Optional dseVersion = ccmRule.getDseVersion(); // creating SASI indexes is broken in DSE 6.8.0 - return dseVersion.isPresent() && dseVersion.get().compareTo(BROKEN_SASI_VERSION) == 0; + return ccmRule.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.compareTo(BROKEN_SASI_VERSION) == 0); } protected static boolean supportsSASI(BaseCcmRule ccmRule) { diff --git a/integration-tests/src/test/resources/DescribeIT/hcd/1.0.cql b/integration-tests/src/test/resources/DescribeIT/hcd/1.0.cql new file mode 100644 index 00000000000..abc70728206 --- /dev/null +++ b/integration-tests/src/test/resources/DescribeIT/hcd/1.0.cql @@ -0,0 +1,186 @@ + +CREATE KEYSPACE ks_0 WITH replication = { 'class' : 'org.apache.cassandra.locator.SimpleStrategy', 'replication_factor': '1' } AND durable_writes = true; + +CREATE TYPE ks_0.btype ( + a text +); + +CREATE TYPE ks_0.xtype ( + d text +); + +CREATE TYPE ks_0.ztype ( + c text, + a int +); + +CREATE TYPE ks_0.ctype ( + z frozen, + x frozen +); + +CREATE TYPE ks_0.atype ( + c frozen +); + +CREATE TABLE ks_0.cyclist_mv ( + cid uuid, + age int, + birthday date, + country text, + name text, + PRIMARY KEY (cid) +) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND default_time_to_live = 0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE INDEX cyclist_by_country ON ks_0.cyclist_mv (country); + +CREATE TABLE ks_0.rank_by_year_and_name ( + race_year int, + race_name text, + rank int, + cyclist_name text, + PRIMARY KEY ((race_year, race_name), rank) +) WITH CLUSTERING ORDER BY (rank DESC) + AND additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND default_time_to_live = 0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE INDEX rrank ON ks_0.rank_by_year_and_name (rank); + +CREATE INDEX ryear ON ks_0.rank_by_year_and_name (race_year); + +CREATE TABLE ks_0.ztable ( + zkey text, + a frozen, + PRIMARY KEY (zkey) +) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.1 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.LeveledCompactionStrategy','max_threshold':'32','min_threshold':'4','sstable_size_in_mb':'95'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND default_time_to_live = 0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE MATERIALIZED VIEW ks_0.cyclist_by_a_age AS +SELECT * FROM ks_0.cyclist_mv +WHERE age IS NOT NULL AND cid IS NOT NULL +PRIMARY KEY (age, cid) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE MATERIALIZED VIEW ks_0.cyclist_by_age AS +SELECT + age, + cid, + birthday, + country, + name +FROM ks_0.cyclist_mv +WHERE age IS NOT NULL AND cid IS NOT NULL +PRIMARY KEY (age, cid) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = 'simple view' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE MATERIALIZED VIEW ks_0.cyclist_by_r_age AS +SELECT + age, + cid, + birthday, + country, + name +FROM ks_0.cyclist_mv +WHERE age IS NOT NULL AND cid IS NOT NULL +PRIMARY KEY (age, cid) WITH additional_write_policy = '99p' + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys':'ALL','rows_per_partition':'NONE'} + AND comment = '' + AND compaction = {'class':'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy','max_threshold':'32','min_threshold':'4'} + AND compression = {'chunk_length_in_kb':'64','class':'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND extensions = {} + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair = 'BLOCKING' + AND speculative_retry = '99p'; + +CREATE FUNCTION ks_0.avgfinal(state tuple) + CALLED ON NULL INPUT + RETURNS double + LANGUAGE java + AS 'double r = 0; if (state.getInt(0) == 0) return null; r = state.getLong(1); r /= state.getInt(0); return Double.valueOf(r);'; + +CREATE FUNCTION ks_0.avgstate(state tuple,val int) + CALLED ON NULL INPUT + RETURNS tuple + LANGUAGE java + AS 'if (val !=null) { state.setInt(0, state.getInt(0)+1); state.setLong(1, state.getLong(1)+val.intValue()); } return state;'; + +CREATE AGGREGATE ks_0.average(int) + SFUNC avgstate + STYPE tuple + FINALFUNC avgfinal + INITCOND (0,0); + +CREATE AGGREGATE ks_0.mean(int) + SFUNC avgstate + STYPE tuple + FINALFUNC avgfinal + INITCOND (0,0); diff --git a/osgi-tests/README.md b/osgi-tests/README.md index 89ad0ba27c8..1ca6211d427 100644 --- a/osgi-tests/README.md +++ b/osgi-tests/README.md @@ -53,8 +53,8 @@ OSGi ones, you can do so as follows: You can pass the following system properties to your tests: 1. `ccm.version`: the CCM version to use -2. `ccm.dse`: whether to use DSE -3. `osgi.debug`: whether to enable remote debugging of the OSGi container (see +2. `ccm.distribution`: choose target backend type (e.g. DSE, HCD) +3. `osgi.debug`: whether to enable remote debugging of the OSGi container (see below). ## Debugging OSGi tests diff --git a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java index 8b140930870..ce4d9095361 100644 --- a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java +++ b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/CcmStagedReactor.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.Version; import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import java.util.List; import java.util.Objects; import net.jcip.annotations.GuardedBy; @@ -38,7 +39,7 @@ public class CcmStagedReactor extends AllConfinedStagedReactor { static { CcmBridge.Builder builder = CcmBridge.builder().withNodes(1); - if (CcmBridge.DSE_ENABLEMENT && CcmBridge.VERSION.compareTo(DSE_5_0) >= 0) { + if (CcmBridge.isDistributionOf(BackendType.DSE, (dist, cass) -> dist.compareTo(DSE_5_0) >= 0)) { builder.withDseWorkloads("graph"); } CCM_BRIDGE = builder.build(); @@ -54,11 +55,10 @@ public CcmStagedReactor(List containers, List m @Override public synchronized void beforeSuite() { if (!running) { - boolean dse = CCM_BRIDGE.getDseVersion().isPresent(); LOGGER.info( "Starting CCM, running {} version {}", - dse ? "DSE" : "Cassandra", - dse ? CCM_BRIDGE.getDseVersion().get() : CCM_BRIDGE.getCassandraVersion()); + CcmBridge.DISTRIBUTION, + CcmBridge.getDistributionVersion()); CCM_BRIDGE.create(); CCM_BRIDGE.start(); LOGGER.info("CCM started"); diff --git a/test-infra/revapi.json b/test-infra/revapi.json index 3cfbc8b5337..c75a98cb4af 100644 --- a/test-infra/revapi.json +++ b/test-infra/revapi.json @@ -171,6 +171,27 @@ "code": "java.method.removed", "old": "method void com.datastax.oss.driver.api.testinfra.ccm.CcmRule::reloadCore(int, java.lang.String, java.lang.String, boolean)", "justification": "Modifying the state of a globally shared CCM instance is dangerous" + }, + { + "code": "java.method.removed", + "old": "method java.util.Optional com.datastax.oss.driver.api.testinfra.ccm.BaseCcmRule::getDseVersion()", + "justification": "Method has been replaced with more generic isDistributionOf(BackendType) and getDistributionVersion()" + }, + { + "code": "java.field.removed", + "old": "field com.datastax.oss.driver.api.testinfra.ccm.CcmBridge.DSE_ENABLEMENT", + "justification": "Method has been replaced with more generic isDistributionOf(BackendType) and getDistributionVersion()" + }, + { + "code": "java.method.nowStatic", + "old": "method com.datastax.oss.driver.api.core.Version com.datastax.oss.driver.api.testinfra.ccm.CcmBridge::getCassandraVersion()", + "new": "method com.datastax.oss.driver.api.core.Version com.datastax.oss.driver.api.testinfra.ccm.CcmBridge::getCassandraVersion()", + "justification": "Previous and current implemntation do not relay on non-static fields" + }, + { + "code": "java.method.removed", + "old": "method java.util.Optional com.datastax.oss.driver.api.testinfra.ccm.CcmBridge::getDseVersion()", + "justification": "Method has been replaced with more generic isDistributionOf(BackendType) and getDistributionVersion()" } ] } diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java index 65210acd2a2..882cd55b948 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/BaseCcmRule.java @@ -22,7 +22,7 @@ import com.datastax.oss.driver.api.core.Version; import com.datastax.oss.driver.api.testinfra.CassandraResourceRule; import com.datastax.oss.driver.api.testinfra.requirement.BackendRequirementRule; -import java.util.Optional; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import org.junit.AssumptionViolatedException; import org.junit.runner.Description; import org.junit.runners.model.Statement; @@ -72,17 +72,29 @@ public void evaluate() { } } - public Version getCassandraVersion() { - return ccmBridge.getCassandraVersion(); + public BackendType getDistribution() { + return CcmBridge.DISTRIBUTION; + } + + public boolean isDistributionOf(BackendType type) { + return CcmBridge.isDistributionOf(type); + } + + public boolean isDistributionOf(BackendType type, CcmBridge.VersionComparator comparator) { + return CcmBridge.isDistributionOf(type, comparator); + } + + public Version getDistributionVersion() { + return CcmBridge.getDistributionVersion(); } - public Optional getDseVersion() { - return ccmBridge.getDseVersion(); + public Version getCassandraVersion() { + return CcmBridge.getCassandraVersion(); } @Override public ProtocolVersion getHighestProtocolVersion() { - if (ccmBridge.getCassandraVersion().compareTo(Version.V2_2_0) >= 0) { + if (CcmBridge.getCassandraVersion().compareTo(Version.V2_2_0) >= 0) { return DefaultProtocolVersion.V4; } else { return DefaultProtocolVersion.V3; diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java index 5b0c114a5fe..f0ce6bc5b0e 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.api.testinfra.ccm; import com.datastax.oss.driver.api.core.Version; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.shaded.guava.common.base.Joiner; import com.datastax.oss.driver.shaded.guava.common.io.Resources; import java.io.File; @@ -54,6 +55,9 @@ public class CcmBridge implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(CcmBridge.class); + public static BackendType DISTRIBUTION = + BackendType.valueOf( + System.getProperty("ccm.distribution", BackendType.CASSANDRA.name()).toUpperCase()); public static final Version VERSION = Objects.requireNonNull(Version.parse(System.getProperty("ccm.version", "4.0.0"))); @@ -61,8 +65,6 @@ public class CcmBridge implements AutoCloseable { public static final String BRANCH = System.getProperty("ccm.branch"); - public static final Boolean DSE_ENABLEMENT = Boolean.getBoolean("ccm.dse"); - public static final String CLUSTER_NAME = "ccm_1"; public static final String DEFAULT_CLIENT_TRUSTSTORE_PASSWORD = "fakePasswordForTests"; @@ -101,22 +103,21 @@ public class CcmBridge implements AutoCloseable { createTempStore(DEFAULT_SERVER_LOCALHOST_KEYSTORE_PATH); // major DSE versions - private static final Version V6_0_0 = Version.parse("6.0.0"); - private static final Version V5_1_0 = Version.parse("5.1.0"); - private static final Version V5_0_0 = Version.parse("5.0.0"); + public static final Version V6_0_0 = Version.parse("6.0.0"); + public static final Version V5_1_0 = Version.parse("5.1.0"); + public static final Version V5_0_0 = Version.parse("5.0.0"); // mapped C* versions from DSE versions - private static final Version V4_0_0 = Version.parse("4.0.0"); - private static final Version V3_10 = Version.parse("3.10"); - private static final Version V3_0_15 = Version.parse("3.0.15"); - private static final Version V2_1_19 = Version.parse("2.1.19"); + public static final Version V4_0_0 = Version.parse("4.0.0"); + public static final Version V3_10 = Version.parse("3.10"); + public static final Version V3_0_15 = Version.parse("3.0.15"); + public static final Version V2_1_19 = Version.parse("2.1.19"); + + // mapped C* versions from HCD versions + public static final Version V4_0_11 = Version.parse("4.0.11"); static { - if (DSE_ENABLEMENT) { - LOG.info("CCM Bridge configured with DSE version {}", VERSION); - } else { - LOG.info("CCM Bridge configured with Apache Cassandra version {}", VERSION); - } + LOG.info("CCM Bridge configured with {} version {}", DISTRIBUTION.getFriendlyName(), VERSION); } private final int[] nodes; @@ -175,25 +176,24 @@ private static boolean isWindows() { return System.getProperty("os.name", "").toLowerCase(Locale.US).contains("win"); } - public Optional getDseVersion() { - return DSE_ENABLEMENT ? Optional.of(VERSION) : Optional.empty(); + public static boolean isDistributionOf(BackendType type) { + return DISTRIBUTION == type; + } + + public static boolean isDistributionOf(BackendType type, VersionComparator comparator) { + return isDistributionOf(type) + && comparator.accept(getDistributionVersion(), getCassandraVersion()); + } + + public static Version getDistributionVersion() { + return VERSION; } - public Version getCassandraVersion() { - if (!DSE_ENABLEMENT) { + public static Version getCassandraVersion() { + if (isDistributionOf(BackendType.CASSANDRA)) { return VERSION; - } else { - Version stableVersion = VERSION.nextStable(); - if (stableVersion.compareTo(V6_0_0) >= 0) { - return V4_0_0; - } else if (stableVersion.compareTo(V5_1_0) >= 0) { - return V3_10; - } else if (stableVersion.compareTo(V5_0_0) >= 0) { - return V3_0_15; - } else { - return V2_1_19; - } } + return DistributionCassandraVersions.getCassandraVersion(DISTRIBUTION, VERSION); } private String getCcmVersionString(Version version) { @@ -225,9 +225,7 @@ public void create() { } else { createOptions.add("-v " + getCcmVersionString(VERSION)); } - if (DSE_ENABLEMENT) { - createOptions.add("--dse"); - } + createOptions.addAll(Arrays.asList(DISTRIBUTION.getCcmOptions())); execute( "create", CLUSTER_NAME, @@ -252,7 +250,7 @@ public void create() { // If we're dealing with anything more recent than 2.2 explicitly enable UDF... but run it // through our conversion process to make // sure more recent versions don't have a problem. - if (cassandraVersion.compareTo(Version.V2_2_0) >= 0) { + if (cassandraVersion.compareTo(Version.V2_2_0) >= 0 || isDistributionOf(BackendType.HCD)) { String originalKey = "enable_user_defined_functions"; Object originalValue = "true"; execute( @@ -264,7 +262,7 @@ public void create() { } // Note that we aren't performing any substitution on DSE key/value props (at least for now) - if (DSE_ENABLEMENT) { + if (isDistributionOf(BackendType.DSE)) { for (Map.Entry conf : dseConfiguration.entrySet()) { execute("updatedseconf", String.format("%s:%s", conf.getKey(), conf.getValue())); } @@ -338,11 +336,10 @@ public void stop(int n) { } public void add(int n, String dc) { - if (getDseVersion().isPresent()) { - execute("add", "-i", ipPrefix + n, "-d", dc, "node" + n, "--dse"); - } else { - execute("add", "-i", ipPrefix + n, "-d", dc, "node" + n); - } + List addOptions = new ArrayList<>(); + addOptions.addAll(Arrays.asList("add", "-i", ipPrefix + n, "-d", dc, "node" + n)); + addOptions.addAll(Arrays.asList(DISTRIBUTION.getCcmOptions())); + execute(addOptions.toArray(new String[0])); start(n); } @@ -475,11 +472,11 @@ private Optional overrideJvmVersionForDseWorkloads() { return Optional.empty(); } - if (!DSE_ENABLEMENT || !getDseVersion().isPresent()) { + if (!isDistributionOf(BackendType.DSE)) { return Optional.empty(); } - if (getDseVersion().get().compareTo(Version.V6_9_0) >= 0) { + if (getDistributionVersion().compareTo(Version.V6_9_0) >= 0) { // DSE 6.9.0 supports only JVM 11 onwards (also with graph workload) return Optional.empty(); } @@ -641,4 +638,8 @@ public CcmBridge build() { dseWorkloads); } } + + public interface VersionComparator { + boolean accept(Version distribution, Version cassandra); + } } diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DefaultCcmBridgeBuilderCustomizer.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DefaultCcmBridgeBuilderCustomizer.java index ac2507cec53..0819f785446 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DefaultCcmBridgeBuilderCustomizer.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DefaultCcmBridgeBuilderCustomizer.java @@ -18,18 +18,20 @@ package com.datastax.oss.driver.api.testinfra.ccm; import com.datastax.oss.driver.api.core.Version; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; /** @see CcmRule */ @SuppressWarnings("unused") public class DefaultCcmBridgeBuilderCustomizer { public static CcmBridge.Builder configureBuilder(CcmBridge.Builder builder) { - if (!CcmBridge.DSE_ENABLEMENT - && CcmBridge.VERSION.nextStable().compareTo(Version.V4_0_0) >= 0) { + if (!CcmBridge.isDistributionOf( + BackendType.DSE, (dist, cass) -> dist.nextStable().compareTo(Version.V4_0_0) >= 0) + || CcmBridge.isDistributionOf(BackendType.HCD)) { builder.withCassandraConfiguration("enable_materialized_views", true); builder.withCassandraConfiguration("enable_sasi_indexes", true); } - if (CcmBridge.VERSION.nextStable().compareTo(Version.V3_0_0) >= 0) { + if (CcmBridge.getDistributionVersion().nextStable().compareTo(Version.V3_0_0) >= 0) { builder.withJvmArgs("-Dcassandra.superuser_setup_delay_ms=0"); builder.withJvmArgs("-Dcassandra.skip_wait_for_gossip_to_settle=0"); builder.withCassandraConfiguration("num_tokens", "1"); diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DistributionCassandraVersions.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DistributionCassandraVersions.java new file mode 100644 index 00000000000..9f7634d1b37 --- /dev/null +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/DistributionCassandraVersions.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.testinfra.ccm; + +import com.datastax.oss.driver.api.core.Version; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableSortedMap; +import java.util.HashMap; +import java.util.Map; + +/** Defines mapping of various distributions to shipped Apache Cassandra version. */ +public abstract class DistributionCassandraVersions { + private static final Map> mappings = + new HashMap<>(); + + static { + { + // DSE + ImmutableSortedMap dse = + ImmutableSortedMap.of( + Version.V1_0_0, CcmBridge.V2_1_19, + Version.V5_0_0, CcmBridge.V3_0_15, + CcmBridge.V5_1_0, CcmBridge.V3_10, + CcmBridge.V6_0_0, CcmBridge.V4_0_0); + mappings.put(BackendType.DSE, dse); + } + { + // HCD + ImmutableSortedMap hcd = + ImmutableSortedMap.of(Version.V1_0_0, CcmBridge.V4_0_11); + mappings.put(BackendType.HCD, hcd); + } + } + + public static Version getCassandraVersion(BackendType type, Version version) { + ImmutableSortedMap mapping = mappings.get(type); + if (mapping == null) { + return null; + } + return mapping.floorEntry(version).getValue(); + } +} diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendRequirementRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendRequirementRule.java index 6c59e216602..343861571e0 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendRequirementRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendRequirementRule.java @@ -41,7 +41,7 @@ public void evaluate() { } protected static BackendType getBackendType() { - return CcmBridge.DSE_ENABLEMENT ? BackendType.DSE : BackendType.CASSANDRA; + return CcmBridge.DISTRIBUTION; } protected static Version getVersion() { diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendType.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendType.java index 1683dd86136..e0058ca324a 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendType.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/requirement/BackendType.java @@ -18,9 +18,9 @@ package com.datastax.oss.driver.api.testinfra.requirement; public enum BackendType { - CASSANDRA("C*"), - DSE("Dse"), - ; + CASSANDRA("Apache Cassandra"), + DSE("DSE"), + HCD("HCD"); final String friendlyName; @@ -31,4 +31,11 @@ public enum BackendType { public String getFriendlyName() { return friendlyName; } + + public String[] getCcmOptions() { + if (this == CASSANDRA) { + return new String[0]; + } + return new String[] {"--" + name().toLowerCase()}; + } } diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java index 5396e5c6cc6..3b792374769 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/session/SessionRule.java @@ -29,10 +29,11 @@ import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.api.testinfra.CassandraResourceRule; import com.datastax.oss.driver.api.testinfra.ccm.BaseCcmRule; +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; import com.datastax.oss.driver.api.testinfra.ccm.SchemaChangeSynchronizer; +import com.datastax.oss.driver.api.testinfra.requirement.BackendType; import com.datastax.oss.driver.api.testinfra.simulacron.SimulacronRule; import java.util.Objects; -import java.util.Optional; import org.junit.rules.ExternalResource; /** @@ -154,14 +155,12 @@ protected void before() { Statement.SYNC); } if (graphName != null) { - Optional dseVersion = - (cassandraResource instanceof BaseCcmRule) - ? ((BaseCcmRule) cassandraResource).getDseVersion() - : Optional.empty(); - if (!dseVersion.isPresent()) { + BaseCcmRule rule = + (cassandraResource instanceof BaseCcmRule) ? ((BaseCcmRule) cassandraResource) : null; + if (rule == null || !CcmBridge.isDistributionOf(BackendType.DSE)) { throw new IllegalArgumentException("DseSessionRule should work with DSE."); } - if (dseVersion.get().compareTo(V6_8_0) >= 0) { + if (rule.getDistributionVersion().compareTo(V6_8_0) >= 0) { session() .execute( ScriptGraphStatement.newInstance( From e84378681aecdbb87696dc4b53cb6fd336c82b6b Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Wed, 23 Oct 2024 22:23:39 +0200 Subject: [PATCH 25/48] Fix TableMetadata.describe() when containing a vector column patch by Stefan Miklosovic; reviewed by Bret McGuire for CASSJAVA-2 --- .../internal/core/type/DefaultVectorType.java | 2 +- .../metadata/schema/TableMetadataTest.java | 67 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/metadata/schema/TableMetadataTest.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java index 5915adc2fb3..c9180d44edc 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java @@ -60,7 +60,7 @@ public String getClassName() { @NonNull @Override public String asCql(boolean includeFrozen, boolean pretty) { - return String.format("'%s(%d)'", getClassName(), getDimensions()); + return String.format("vector<%s, %d>", getElementType().asCql(true, false), getDimensions()); } /* ============== General class implementation ============== */ diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/schema/TableMetadataTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/schema/TableMetadataTest.java new file mode 100644 index 00000000000..03d63230992 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/schema/TableMetadataTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata.schema; + +import static com.datastax.oss.driver.api.core.CqlIdentifier.fromCql; +import static org.assertj.core.api.Assertions.assertThat; + +import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.datastax.oss.driver.internal.core.type.DefaultVectorType; +import com.datastax.oss.driver.internal.core.type.PrimitiveType; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import com.datastax.oss.protocol.internal.ProtocolConstants.DataType; +import com.google.common.collect.ImmutableList; +import java.util.UUID; +import org.junit.Test; + +public class TableMetadataTest { + + /** Tests CASSJAVA-2 */ + @Test + public void should_describe_table_with_vector_correctly() { + TableMetadata tableMetadata = + new DefaultTableMetadata( + fromCql("ks"), + fromCql("tb"), + UUID.randomUUID(), + false, + false, + ImmutableList.of( + new DefaultColumnMetadata( + fromCql("ks"), + fromCql("ks"), + fromCql("tb"), + new PrimitiveType(DataType.ASCII), + false)), + ImmutableMap.of(), + ImmutableMap.of( + fromCql("a"), + new DefaultColumnMetadata( + fromCql("ks"), + fromCql("ks"), + fromCql("tb"), + new DefaultVectorType(new PrimitiveType(DataType.INT), 3), + false)), + ImmutableMap.of(), + ImmutableMap.of()); + + String describe1 = tableMetadata.describe(true); + + assertThat(describe1).contains("vector,"); + } +} From 62cea5a5f34d5cc6ef335f99829d0ae3cf9cf396 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Wed, 6 Nov 2024 17:47:30 -0600 Subject: [PATCH 26/48] Move Apache Cassandra 5.x off of beta1 and remove some older Apache Cassandra versions. patch by Bret McGuire; reviewed by Bret McGuire for CASSJAVA-54 --- Jenkinsfile-datastax | 52 +++++++++++++------------------------------- 1 file changed, 15 insertions(+), 37 deletions(-) diff --git a/Jenkinsfile-datastax b/Jenkinsfile-datastax index af3faafee20..cd48f325a29 100644 --- a/Jenkinsfile-datastax +++ b/Jenkinsfile-datastax @@ -268,13 +268,9 @@ pipeline { ''') choice( name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_SERVER_VERSION', - choices: ['2.1', // Legacy Apache CassandraⓇ - '2.2', // Legacy Apache CassandraⓇ - '3.0', // Previous Apache CassandraⓇ - '3.11', // Previous Apache CassandraⓇ - '4.0', // Previous Apache CassandraⓇ - '4.1', // Current Apache CassandraⓇ - '5.0-beta1', // Development Apache CassandraⓇ + choices: ['4.0', // Previous Apache CassandraⓇ + '4.1', // Previous Apache CassandraⓇ + '5.0', // Current Apache CassandraⓇ 'dse-4.8.16', // Previous EOSL DataStax Enterprise 'dse-5.0.15', // Long Term Support DataStax Enterprise 'dse-5.1.35', // Legacy DataStax Enterprise @@ -292,22 +288,6 @@ pipeline { Choice Description - - 2.1 - Apache Cassandra® v2.1.x - - - 2.2 - Apache Cassandra® v2.2.x - - - 3.0 - Apache Cassandra® v3.0.x - - - 3.11 - Apache Cassandra® v3.11.x - 4.0 Apache Cassandra® v4.0.x @@ -316,6 +296,10 @@ pipeline { 4.1 Apache Cassandra® v4.1.x + + 5.0 + Apache Cassandra® v5.0.x + dse-4.8.16 DataStax Enterprise v4.8.x (END OF SERVICE LIFE) @@ -435,13 +419,10 @@ pipeline { // schedules only run against release branches (i.e. 3.x, 4.x, 4.5.x, etc.) parameterizedCron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? """ # Every weekend (Saturday, Sunday) around 2:00 AM - ### JDK8 tests against 2.1, 3.0, 4.0, DSE 4.8, DSE 5.0, DSE 5.1, dse-6.0.18 and DSE 6.7 - H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=2.1 3.0 4.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 + H 2 * * 0 %CI_SCHEDULE=WEEKENDS;CI_SCHEDULE_SERVER_VERSIONS=4.0 4.1 5.0 dse-4.8.16 dse-5.0.15 dse-5.1.35 dse-6.0.18 dse-6.7.17;CI_SCHEDULE_JABBA_VERSION=1.8 # Every weeknight (Monday - Friday) around 12:00 PM noon - ### JDK11 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 - ### JDK17 tests against 3.11, 4.1, 5.0-beta1 and DSE 6.8 - H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=3.11 4.1 5.0-beta1 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=4.1 5.0 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.11 + H 12 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;CI_SCHEDULE_SERVER_VERSIONS=4.1 5.0 dse-6.8.30 dse-6.9.0 hcd-1.0.0;CI_SCHEDULE_JABBA_VERSION=openjdk@1.17 """ : "") } @@ -475,8 +456,8 @@ pipeline { axes { axis { name 'SERVER_VERSION' - values '3.11', // Latest stable Apache CassandraⓇ - '4.1', // Development Apache CassandraⓇ + values '4.0', // Previous Apache CassandraⓇ + '5.0', // Current Apache CassandraⓇ 'dse-6.8.30', // Current DataStax Enterprise 'dse-6.9.0', // Current DataStax Enterprise 'hcd-1.0.0' // Current DataStax HCD @@ -585,12 +566,9 @@ pipeline { axes { axis { name 'SERVER_VERSION' - values '2.1', // Legacy Apache CassandraⓇ - '3.0', // Previous Apache CassandraⓇ - '3.11', // Previous Apache CassandraⓇ - '4.0', // Previous Apache CassandraⓇ - '4.1', // Current Apache CassandraⓇ - '5.0-beta1', // Development Apache CassandraⓇ + values '4.0', // Previous Apache CassandraⓇ + '4.1', // Previous Apache CassandraⓇ + '5.0', // Current Apache CassandraⓇ 'dse-4.8.16', // Previous EOSL DataStax Enterprise 'dse-5.0.15', // Last EOSL DataStax Enterprise 'dse-5.1.35', // Legacy DataStax Enterprise From a322ca265654605123f4d7b889ad736c114f0c7e Mon Sep 17 00:00:00 2001 From: Jeremy Hanna Date: Wed, 27 Nov 2024 12:41:02 -0600 Subject: [PATCH 27/48] Update link to Jira to be CASSJAVA Updating the link to Jira. Previously we had a component in the CASSANDRA Jira project but now we have a project for each driver - in the case of Java, it's CASSJAVA. Added CASSJAVA to .asf.yaml patch by Jeremy Hanna; reviewed by Bret McGuire for CASSJAVA-61 --- .asf.yaml | 1 + README.md | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index ad58f536398..ac29efed9ff 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -34,3 +34,4 @@ github: projects: false autolink_jira: - CASSANDRA + - CASSJAVA diff --git a/README.md b/README.md index 2a30cb68c9a..b6e1cc337d8 100644 --- a/README.md +++ b/README.md @@ -75,13 +75,13 @@ See the [Cassandra error handling done right blog](https://www.datastax.com/blog * [Manual](manual/) * [API docs] -* Bug tracking: [JIRA]. Make sure to select the "Client/java-driver" component when filing new tickets! +* Bug tracking: [JIRA] * [Mailing list] * [Changelog] * [FAQ] [API docs]: https://docs.datastax.com/en/drivers/java/4.17 -[JIRA]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20CASSANDRA%20AND%20component%20%3D%20%22Client%2Fjava-driver%22%20ORDER%20BY%20key%20DESC +[JIRA]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20CASSJAVA%20ORDER%20BY%20key%20DESC [Mailing list]: https://groups.google.com/a/lists.datastax.com/forum/#!forum/java-driver-user [Changelog]: changelog/ [FAQ]: faq/ @@ -108,4 +108,4 @@ Apache Cassandra, Apache, Tomcat, Lucene, Solr, Hadoop, Spark, TinkerPop, and Ca trademarks of the [Apache Software Foundation](http://www.apache.org/) or its subsidiaries in Canada, the United States and/or other countries. -Binary artifacts of this product bundle Java Native Runtime libraries, which is available under the Eclipse Public License version 2.0. \ No newline at end of file +Binary artifacts of this product bundle Java Native Runtime libraries, which is available under the Eclipse Public License version 2.0. From 7bc085bdfb337b91255573bc3e130815e280954a Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Wed, 6 Nov 2024 08:31:36 +0100 Subject: [PATCH 28/48] Move DataStax shaded Guava module into Java driver patch by Lukasz Antoniak; reviewed by Alexandre Dutra and Bret McGuire for CASSJAVA-52 --- bom/pom.xml | 10 +- core-shaded/pom.xml | 4 +- core/pom.xml | 4 +- distribution/src/assembly/binary-tarball.xml | 6 +- guava-shaded/pom.xml | 215 ++++++++++++++++++ guava-shaded/src/assembly/shaded-jar.xml | 48 ++++ ...graphicalComparatorHolderSubstitution.java | 39 ++++ .../UnsafeComparatorSubstitution.java | 25 ++ guava-shaded/src/main/javadoc/README.txt | 2 + manual/core/integration/README.md | 2 +- mapper-processor/pom.xml | 4 +- osgi-tests/pom.xml | 4 +- .../internal/osgi/support/BundleOptions.java | 2 +- pom.xml | 3 +- query-builder/pom.xml | 4 +- 15 files changed, 351 insertions(+), 21 deletions(-) create mode 100644 guava-shaded/pom.xml create mode 100644 guava-shaded/src/assembly/shaded-jar.xml create mode 100644 guava-shaded/src/main/java/com/google/common/primitives/LexicographicalComparatorHolderSubstitution.java create mode 100644 guava-shaded/src/main/java/com/google/common/primitives/UnsafeComparatorSubstitution.java create mode 100644 guava-shaded/src/main/javadoc/README.txt diff --git a/bom/pom.xml b/bom/pom.xml index 96b7a6ceb18..08f212f6157 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -55,6 +55,11 @@ java-driver-query-builder 4.18.2-SNAPSHOT + + org.apache.cassandra + java-driver-guava-shaded + 4.18.2-SNAPSHOT + org.apache.cassandra java-driver-test-infra @@ -75,11 +80,6 @@ native-protocol 1.5.1 - - com.datastax.oss - java-driver-shaded-guava - 25.1-jre-graal-sub-1 - diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 6c139aab127..9a708beb2a7 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -57,8 +57,8 @@ native-protocol - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded com.typesafe diff --git a/core/pom.xml b/core/pom.xml index 33688754f1b..2a48e8bf9ce 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -49,8 +49,8 @@ netty-handler - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded com.typesafe diff --git a/distribution/src/assembly/binary-tarball.xml b/distribution/src/assembly/binary-tarball.xml index 0d025fafb2c..b6294a25340 100644 --- a/distribution/src/assembly/binary-tarball.xml +++ b/distribution/src/assembly/binary-tarball.xml @@ -66,8 +66,8 @@ org.apache.cassandra:java-driver-core org.apache.cassandra:java-driver-mapper-runtime org.apache.cassandra:java-driver-mapper-processor + org.apache.cassandra:java-driver-guava-shaded - com.datastax.oss:java-driver-shaded-guava com.github.stephenc.jcip:jcip-annotations com.github.spotbugs:spotbugs-annotations @@ -91,8 +91,8 @@ org.apache.cassandra:java-driver-core org.apache.cassandra:java-driver-query-builder org.apache.cassandra:java-driver-mapper-processor + org.apache.cassandra:java-driver-guava-shaded - com.datastax.oss:java-driver-shaded-guava com.github.stephenc.jcip:jcip-annotations com.github.spotbugs:spotbugs-annotations @@ -116,8 +116,8 @@ org.apache.cassandra:java-driver-core org.apache.cassandra:java-driver-query-builder org.apache.cassandra:java-driver-mapper-runtime + org.apache.cassandra:java-driver-guava-shaded - com.datastax.oss:java-driver-shaded-guava com.github.stephenc.jcip:jcip-annotations com.github.spotbugs:spotbugs-annotations diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml new file mode 100644 index 00000000000..9854fcc48ba --- /dev/null +++ b/guava-shaded/pom.xml @@ -0,0 +1,215 @@ + + + + 4.0.0 + + org.apache.cassandra + java-driver-parent + 4.18.2-SNAPSHOT + + java-driver-guava-shaded + Apache Cassandra Java Driver - guava shaded dep + Shaded Guava artifact for use in the Java driver for Apache Cassandra® + + + com.google.guava + guava + + + com.google.code.findbugs + jsr305 + + + org.checkerframework + checker-qual + + + com.google.errorprone + error_prone_annotations + + + com.google.j2objc + j2objc-annotations + + + org.codehaus.mojo + animal-sniffer-annotations + + + + + org.graalvm.nativeimage + svm + 20.0.0 + provided + + + + + + maven-shade-plugin + + + shade-guava-dependency + package + + shade + + + + + org.apache.cassandra:java-driver-guava-shaded + com.google.guava:guava + + + + + com.google + com.datastax.oss.driver.shaded.guava + + + + + com.google.guava:* + + META-INF/** + + + + true + true + + + + + + maven-clean-plugin + + + clean-classes + package + + clean + + + ${project.build.outputDirectory} + + + + + + maven-dependency-plugin + + + unpack-shaded-classes + package + + unpack + + + ${project.build.outputDirectory} + + + org.apache.cassandra + java-driver-guava-shaded + ${project.version} + jar + + + + + + + + org.apache.felix + maven-bundle-plugin + + 3.5.0 + true + + + generate-shaded-manifest + package + + manifest + + + + com.datastax.oss.driver.shaded.guava + !com.datastax.oss.driver.shaded.guava.errorprone.*, !org.checkerframework.*, * + javax.annotation.*;resolution:=optional;version="[3.0,4)", javax.crypto.*;resolution:=optional, sun.misc.*;resolution:=optional, !com.oracle.svm.*, !com.datastax.oss.driver.shaded.guava.errorprone.*, !org.checkerframework.*, * + + + + + + + maven-assembly-plugin + + + generate-final-shaded-jar + package + + single + + + + + ${project.build.outputDirectory}/META-INF/MANIFEST.MF + + + src/assembly/shaded-jar.xml + + + false + + + + + + maven-jar-plugin + + + empty-javadoc-jar + + jar + + + javadoc + ${basedir}/src/main/javadoc + + + + + + org.revapi + revapi-maven-plugin + + true + + + + + diff --git a/guava-shaded/src/assembly/shaded-jar.xml b/guava-shaded/src/assembly/shaded-jar.xml new file mode 100644 index 00000000000..d762a27b20f --- /dev/null +++ b/guava-shaded/src/assembly/shaded-jar.xml @@ -0,0 +1,48 @@ + + + + shaded-jar + + jar + + false + + + + ${project.build.outputDirectory} + + META-INF/maven/org.apache.cassandra/java-driver-guava-shaded/pom.xml + + + + + + + + ${project.basedir}/dependency-reduced-pom.xml + META-INF/maven/org.apache.cassandra/java-driver-guava-shaded + pom.xml + + + diff --git a/guava-shaded/src/main/java/com/google/common/primitives/LexicographicalComparatorHolderSubstitution.java b/guava-shaded/src/main/java/com/google/common/primitives/LexicographicalComparatorHolderSubstitution.java new file mode 100644 index 00000000000..95e9c70cdbc --- /dev/null +++ b/guava-shaded/src/main/java/com/google/common/primitives/LexicographicalComparatorHolderSubstitution.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.common.primitives; + +import com.oracle.svm.core.annotate.Alias; +import com.oracle.svm.core.annotate.RecomputeFieldValue; +import com.oracle.svm.core.annotate.Substitute; +import com.oracle.svm.core.annotate.TargetClass; +import java.util.Comparator; + +@TargetClass(UnsignedBytes.LexicographicalComparatorHolder.class) +final class LexicographicalComparatorHolderSubstitution { + + @Alias + @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.FromAlias) + static Comparator BEST_COMPARATOR = UnsignedBytes.lexicographicalComparatorJavaImpl(); + + /* All known cases should be covered by the field substitution above... keeping this only + * for sake of completeness */ + @Substitute + static Comparator getBestComparator() { + return UnsignedBytes.lexicographicalComparatorJavaImpl(); + } +} diff --git a/guava-shaded/src/main/java/com/google/common/primitives/UnsafeComparatorSubstitution.java b/guava-shaded/src/main/java/com/google/common/primitives/UnsafeComparatorSubstitution.java new file mode 100644 index 00000000000..549de0b5c02 --- /dev/null +++ b/guava-shaded/src/main/java/com/google/common/primitives/UnsafeComparatorSubstitution.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.common.primitives; + +import com.oracle.svm.core.annotate.Delete; +import com.oracle.svm.core.annotate.TargetClass; + +@TargetClass(UnsignedBytes.LexicographicalComparatorHolder.UnsafeComparator.class) +@Delete +final class UnsafeComparatorSubstitution {} diff --git a/guava-shaded/src/main/javadoc/README.txt b/guava-shaded/src/main/javadoc/README.txt new file mode 100644 index 00000000000..57f82b2a265 --- /dev/null +++ b/guava-shaded/src/main/javadoc/README.txt @@ -0,0 +1,2 @@ +This empty JAR is generated for compliance with Maven Central rules. Please refer to the original +Guava API docs. \ No newline at end of file diff --git a/manual/core/integration/README.md b/manual/core/integration/README.md index 2dfc0155c63..f2a96160bce 100644 --- a/manual/core/integration/README.md +++ b/manual/core/integration/README.md @@ -671,7 +671,7 @@ The remaining core driver dependencies are the only ones that are truly mandator * the [native protocol](https://github.com/datastax/native-protocol) layer. This is essentially part of the driver code, but was externalized for reuse in other projects; -* `java-driver-shaded-guava`, a shaded version of [Guava](https://github.com/google/guava). It is +* `java-driver-guava-shaded`, a shaded version of [Guava](https://github.com/google/guava). It is relocated to a different package, and only used by internal driver code, so it should be completely transparent to third-party code; * the [SLF4J](https://www.slf4j.org/) API for [logging](../logging/). diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index 61906f41987..6588f17d5f7 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -44,8 +44,8 @@ java-driver-mapper-runtime - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded com.squareup diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index 5947aff1bc5..f0e66b656ca 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -71,8 +71,8 @@ logback-classic - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded org.xerial.snappy diff --git a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java index 536a6d96c77..3e6171ca530 100644 --- a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java +++ b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java @@ -35,7 +35,7 @@ public class BundleOptions { public static CompositeOption commonBundles() { return () -> options( - mavenBundle("com.datastax.oss", "java-driver-shaded-guava").versionAsInProject(), + mavenBundle("org.apache.cassandra", "java-driver-guava-shaded").versionAsInProject(), mavenBundle("io.dropwizard.metrics", "metrics-core").versionAsInProject(), mavenBundle("org.slf4j", "slf4j-api").versionAsInProject(), mavenBundle("org.hdrhistogram", "HdrHistogram").versionAsInProject(), diff --git a/pom.xml b/pom.xml index 94311719e5f..620cf1db4bb 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ mapper-processor metrics/micrometer metrics/microprofile + guava-shaded test-infra integration-tests osgi-tests @@ -110,7 +111,7 @@ ${netty.version} - + com.google.guava guava 25.1-jre diff --git a/query-builder/pom.xml b/query-builder/pom.xml index bae0e0c6ca0..4e09a10e584 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -45,8 +45,8 @@ java-driver-core - com.datastax.oss - java-driver-shaded-guava + org.apache.cassandra + java-driver-guava-shaded com.github.stephenc.jcip From 7ca013fbd5f1ec589df52ef0e6441986f07c11ff Mon Sep 17 00:00:00 2001 From: Ammar Khaku Date: Sat, 22 Apr 2023 16:13:15 -0700 Subject: [PATCH 29/48] JAVA-3057 Allow decoding a UDT that has more fields than expected patch by Ammar Khaku; reviewed by Andy Tolbert and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1635 --- .../internal/core/type/codec/UdtCodec.java | 10 ++- .../core/type/codec/UdtCodecTest.java | 24 +++--- .../internal/core/type/codec/UdtCodecIT.java | 77 +++++++++++++++++++ 3 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 integration-tests/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecIT.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodec.java index f5177e63b5e..5d0a379f761 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodec.java @@ -30,10 +30,14 @@ import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import net.jcip.annotations.ThreadSafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @ThreadSafe public class UdtCodec implements TypeCodec { + private static final Logger LOG = LoggerFactory.getLogger(UdtCodec.class); + private final UserDefinedType cqlType; public UdtCodec(@NonNull UserDefinedType cqlType) { @@ -107,10 +111,8 @@ public UdtValue decode(@Nullable ByteBuffer bytes, @NonNull ProtocolVersion prot int i = 0; while (input.hasRemaining()) { if (i == cqlType.getFieldTypes().size()) { - throw new IllegalArgumentException( - String.format( - "Too many fields in encoded UDT value, expected %d", - cqlType.getFieldTypes().size())); + LOG.debug("Encountered unexpected fields when parsing codec {}", cqlType); + break; } int elementSize = input.getInt(); ByteBuffer element; diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecTest.java index bf7c1e98b26..af94247f937 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecTest.java @@ -136,18 +136,18 @@ public void should_decode_udt() { } @Test - public void should_fail_to_decode_udt_when_too_many_fields() { - assertThatThrownBy( - () -> - decode( - "0x" - + ("00000004" + "00000001") - + "ffffffff" - + ("00000001" + "61") - // extra contents - + "ffffffff")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Too many fields in encoded UDT value, expected 3"); + public void should_decode_udt_when_too_many_fields() { + UdtValue udt = + decode( + "0x" + + ("00000004" + "00000001") + + "ffffffff" + + ("00000001" + "61") + // extra contents + + "ffffffff"); + assertThat(udt.getInt(0)).isEqualTo(1); + assertThat(udt.isNull(1)).isTrue(); + assertThat(udt.getString(2)).isEqualTo("a"); } /** Test for JAVA-2557. Ensures that the codec can decode null fields with any negative length. */ diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecIT.java new file mode 100644 index 00000000000..804a078bbe0 --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/internal/core/type/codec/UdtCodecIT.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.type.codec; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.data.UdtValue; +import com.datastax.oss.driver.api.core.type.UserDefinedType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.testinfra.ccm.CcmRule; +import com.datastax.oss.driver.api.testinfra.session.SessionRule; +import com.datastax.oss.driver.categories.ParallelizableTests; +import java.util.Objects; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +@Category(ParallelizableTests.class) +public class UdtCodecIT { + + private CcmRule ccmRule = CcmRule.getInstance(); + + private SessionRule sessionRule = SessionRule.builder(ccmRule).build(); + + @Rule public TestRule chain = RuleChain.outerRule(ccmRule).around(sessionRule); + + @Test + public void should_decoding_udt_be_backward_compatible() { + CqlSession session = sessionRule.session(); + session.execute("CREATE TYPE test_type_1 (a text, b int)"); + session.execute("CREATE TABLE test_table_1 (e int primary key, f frozen)"); + // insert a row using version 1 of the UDT schema + session.execute("INSERT INTO test_table_1(e, f) VALUES(1, {a: 'a', b: 1})"); + UserDefinedType udt = + session + .getMetadata() + .getKeyspace(sessionRule.keyspace()) + .flatMap(ks -> ks.getUserDefinedType("test_type_1")) + .orElseThrow(IllegalStateException::new); + TypeCodec oldCodec = session.getContext().getCodecRegistry().codecFor(udt); + // update UDT schema + session.execute("ALTER TYPE test_type_1 add i text"); + // insert a row using version 2 of the UDT schema + session.execute("INSERT INTO test_table_1(e, f) VALUES(2, {a: 'b', b: 2, i: 'b'})"); + Row row = + Objects.requireNonNull(session.execute("SELECT f FROM test_table_1 WHERE e = ?", 2).one()); + // Try to read new row with old codec. Using row.getUdtValue() would not cause any issues, + // because new codec will be automatically registered (using all 3 attributes). + // If application leverages generic row.get(String, Codec) method, data reading with old codec + // should + // be backward-compatible. + UdtValue value = Objects.requireNonNull((UdtValue) row.get("f", oldCodec)); + assertThat(value.getString("a")).isEqualTo("b"); + assertThat(value.getInt("b")).isEqualTo(2); + assertThatThrownBy(() -> value.getString("i")).hasMessage("i is not a field in this UDT"); + } +} From 6a8674f2db92668359196b5492753612b3844594 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Mon, 11 Nov 2024 11:49:20 -0600 Subject: [PATCH 30/48] CASSJAVA-55 Remove setting "Host" header for metadata requests. With some sysprops enabled this will actually be respected which completely borks Astra routing. patch by Bret McGuire; reviewed by Alexandre Dutra and Bret McGuire for CASSJAVA-55 --- .../driver/internal/core/config/cloud/CloudConfigFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/config/cloud/CloudConfigFactory.java b/core/src/main/java/com/datastax/oss/driver/internal/core/config/cloud/CloudConfigFactory.java index b6b2cccc466..817b3263d25 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/config/cloud/CloudConfigFactory.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/config/cloud/CloudConfigFactory.java @@ -229,7 +229,6 @@ protected BufferedReader fetchProxyMetadata( HttpsURLConnection connection = (HttpsURLConnection) metadataServiceUrl.openConnection(); connection.setSSLSocketFactory(sslContext.getSocketFactory()); connection.setRequestMethod("GET"); - connection.setRequestProperty("host", "localhost"); return new BufferedReader( new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8)); } catch (ConnectException e) { From 7689c5a81e89bce5598d9976a041068a1f5e2a7f Mon Sep 17 00:00:00 2001 From: SiyaoIsHiding <113857408+SiyaoIsHiding@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:30:58 +0800 Subject: [PATCH 31/48] JAVA-3118: Add support for vector data type in Schema Builder, QueryBuilder patch by Jane He; reviewed by Mick Semb Wever and Bret McGuire for JAVA-3118 reference: #1931 --- manual/query_builder/select/README.md | 23 +++++ query-builder/revapi.json | 10 +++ .../api/querybuilder/select/Select.java | 11 +++ .../querybuilder/select/DefaultSelect.java | 86 +++++++++++++++++-- .../delete/DeleteSelectorTest.java | 11 +++ .../insert/RegularInsertTest.java | 7 ++ .../querybuilder/schema/AlterTableTest.java | 6 ++ .../querybuilder/schema/AlterTypeTest.java | 6 ++ .../querybuilder/schema/CreateTableTest.java | 9 ++ .../querybuilder/schema/CreateTypeTest.java | 9 ++ .../select/SelectOrderingTest.java | 20 +++++ .../select/SelectSelectorTest.java | 43 ++++++++++ 12 files changed, 233 insertions(+), 8 deletions(-) diff --git a/manual/query_builder/select/README.md b/manual/query_builder/select/README.md index 92c058608e7..0425423a402 100644 --- a/manual/query_builder/select/README.md +++ b/manual/query_builder/select/README.md @@ -387,6 +387,29 @@ selectFrom("sensor_data") // SELECT reading FROM sensor_data WHERE id=? ORDER BY date DESC ``` +Vector Search: + +```java + +import com.datastax.oss.driver.api.core.data.CqlVector; + +selectFrom("foo") + .all() + .where(Relation.column("k").isEqualTo(literal(1))) + .orderByAnnOf("c1", CqlVector.newInstance(0.1, 0.2, 0.3)); +// SELECT * FROM foo WHERE k=1 ORDER BY c1 ANN OF [0.1, 0.2, 0.3] + +selectFrom("cycling", "comments_vs") + .column("comment") + .function( + "similarity_cosine", + Selector.column("comment_vector"), + literal(CqlVector.newInstance(0.2, 0.15, 0.3, 0.2, 0.05))) + .orderByAnnOf("comment_vector", CqlVector.newInstance(0.1, 0.15, 0.3, 0.12, 0.05)) + .limit(1); +// SELECT comment,similarity_cosine(comment_vector,[0.2, 0.15, 0.3, 0.2, 0.05]) FROM cycling.comments_vs ORDER BY comment_vector ANN OF [0.1, 0.15, 0.3, 0.12, 0.05] LIMIT 1 +``` + Limits: ```java diff --git a/query-builder/revapi.json b/query-builder/revapi.json index 9d0163b487e..c4d8aa27212 100644 --- a/query-builder/revapi.json +++ b/query-builder/revapi.json @@ -2772,6 +2772,16 @@ "code": "java.method.addedToInterface", "new": "method com.datastax.oss.driver.api.querybuilder.update.UpdateStart com.datastax.oss.driver.api.querybuilder.update.UpdateStart::usingTtl(int)", "justification": "JAVA-2210: Add ability to set TTL for modification queries" + }, + { + "code": "java.method.addedToInterface", + "new": "method com.datastax.oss.driver.api.querybuilder.select.Select com.datastax.oss.driver.api.querybuilder.select.Select::orderByAnnOf(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector)", + "justification": "JAVA-3118: Add support for vector data type in Schema Builder, QueryBuilder" + }, + { + "code": "java.method.addedToInterface", + "new": "method com.datastax.oss.driver.api.querybuilder.select.Select com.datastax.oss.driver.api.querybuilder.select.Select::orderByAnnOf(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector)", + "justification": "JAVA-3118: Add support for vector data type in Schema Builder, QueryBuilder" } ] } diff --git a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/select/Select.java b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/select/Select.java index a22b45c35bd..159657989da 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/select/Select.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/api/querybuilder/select/Select.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.api.querybuilder.select; import com.datastax.oss.driver.api.core.CqlIdentifier; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; import com.datastax.oss.driver.api.querybuilder.BindMarker; import com.datastax.oss.driver.api.querybuilder.BuildableQuery; @@ -146,6 +147,16 @@ default Select orderBy(@NonNull String columnName, @NonNull ClusteringOrder orde return orderBy(CqlIdentifier.fromCql(columnName), order); } + /** + * Shortcut for {@link #orderByAnnOf(CqlIdentifier, CqlVector)}, adding an ORDER BY ... ANN OF ... + * clause + */ + @NonNull + Select orderByAnnOf(@NonNull String columnName, @NonNull CqlVector ann); + + /** Adds the ORDER BY ... ANN OF ... clause, usually used for vector search */ + @NonNull + Select orderByAnnOf(@NonNull CqlIdentifier columnId, @NonNull CqlVector ann); /** * Adds a LIMIT clause to this query with a literal value. * diff --git a/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/select/DefaultSelect.java b/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/select/DefaultSelect.java index 86a2a07a3f2..5daf252a9eb 100644 --- a/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/select/DefaultSelect.java +++ b/query-builder/src/main/java/com/datastax/oss/driver/internal/querybuilder/select/DefaultSelect.java @@ -20,8 +20,10 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.cql.SimpleStatementBuilder; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.core.metadata.schema.ClusteringOrder; import com.datastax.oss.driver.api.querybuilder.BindMarker; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; import com.datastax.oss.driver.api.querybuilder.relation.Relation; import com.datastax.oss.driver.api.querybuilder.select.Select; import com.datastax.oss.driver.api.querybuilder.select.SelectFrom; @@ -49,6 +51,7 @@ public class DefaultSelect implements SelectFrom, Select { private final ImmutableList relations; private final ImmutableList groupByClauses; private final ImmutableMap orderings; + private final Ann ann; private final Object limit; private final Object perPartitionLimit; private final boolean allowsFiltering; @@ -65,6 +68,7 @@ public DefaultSelect(@Nullable CqlIdentifier keyspace, @NonNull CqlIdentifier ta ImmutableMap.of(), null, null, + null, false); } @@ -74,6 +78,8 @@ public DefaultSelect(@Nullable CqlIdentifier keyspace, @NonNull CqlIdentifier ta * @param selectors if it contains {@link AllSelector#INSTANCE}, that must be the only element. * This isn't re-checked because methods that call this constructor internally already do it, * make sure you do it yourself. + * @param ann Approximate nearest neighbor. ANN ordering does not support secondary ordering or + * ASC order. */ public DefaultSelect( @Nullable CqlIdentifier keyspace, @@ -84,6 +90,7 @@ public DefaultSelect( @NonNull ImmutableList relations, @NonNull ImmutableList groupByClauses, @NonNull ImmutableMap orderings, + @Nullable Ann ann, @Nullable Object limit, @Nullable Object perPartitionLimit, boolean allowsFiltering) { @@ -94,6 +101,9 @@ public DefaultSelect( || (limit instanceof Integer && (Integer) limit > 0) || limit instanceof BindMarker, "limit must be a strictly positive integer or a bind marker"); + Preconditions.checkArgument( + orderings.isEmpty() || ann == null, "ANN ordering does not support secondary ordering"); + this.ann = ann; this.keyspace = keyspace; this.table = table; this.isJson = isJson; @@ -117,6 +127,7 @@ public SelectFrom json() { relations, groupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -134,6 +145,7 @@ public SelectFrom distinct() { relations, groupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -193,6 +205,7 @@ public Select withSelectors(@NonNull ImmutableList newSelectors) { relations, groupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -221,6 +234,7 @@ public Select withRelations(@NonNull ImmutableList newRelations) { newRelations, groupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -249,6 +263,7 @@ public Select withGroupByClauses(@NonNull ImmutableList newGroupByClau relations, newGroupByClauses, orderings, + ann, limit, perPartitionLimit, allowsFiltering); @@ -260,6 +275,18 @@ public Select orderBy(@NonNull CqlIdentifier columnId, @NonNull ClusteringOrder return withOrderings(ImmutableCollections.append(orderings, columnId, order)); } + @NonNull + @Override + public Select orderByAnnOf(@NonNull String columnName, @NonNull CqlVector ann) { + return withAnn(new Ann(CqlIdentifier.fromCql(columnName), ann)); + } + + @NonNull + @Override + public Select orderByAnnOf(@NonNull CqlIdentifier columnId, @NonNull CqlVector ann) { + return withAnn(new Ann(columnId, ann)); + } + @NonNull @Override public Select orderByIds(@NonNull Map newOrderings) { @@ -277,6 +304,24 @@ public Select withOrderings(@NonNull ImmutableMap entry : orderings.entrySet()) { - if (first) { - builder.append(" ORDER BY "); - first = false; - } else { - builder.append(","); + if (ann != null) { + builder.append(" ORDER BY ").append(this.ann.columnId.asCql(true)).append(" ANN OF "); + QueryBuilder.literal(ann.vector).appendTo(builder); + } else { + boolean first = true; + for (Map.Entry entry : orderings.entrySet()) { + if (first) { + builder.append(" ORDER BY "); + first = false; + } else { + builder.append(","); + } + builder.append(entry.getKey().asCql(true)).append(" ").append(entry.getValue().name()); } - builder.append(entry.getKey().asCql(true)).append(" ").append(entry.getValue().name()); } if (limit != null) { @@ -499,6 +554,11 @@ public Object getLimit() { return limit; } + @Nullable + public Ann getAnn() { + return ann; + } + @Nullable public Object getPerPartitionLimit() { return perPartitionLimit; @@ -512,4 +572,14 @@ public boolean allowsFiltering() { public String toString() { return asCql(); } + + public static class Ann { + private final CqlVector vector; + private final CqlIdentifier columnId; + + private Ann(CqlIdentifier columnId, CqlVector vector) { + this.vector = vector; + this.columnId = columnId; + } + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/delete/DeleteSelectorTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/delete/DeleteSelectorTest.java index 23210971bc6..cce4cf51a10 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/delete/DeleteSelectorTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/delete/DeleteSelectorTest.java @@ -22,6 +22,7 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.deleteFrom; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; +import com.datastax.oss.driver.api.core.data.CqlVector; import org.junit.Test; public class DeleteSelectorTest { @@ -34,6 +35,16 @@ public void should_generate_column_deletion() { .hasCql("DELETE v FROM ks.foo WHERE k=?"); } + @Test + public void should_generate_vector_deletion() { + assertThat( + deleteFrom("foo") + .column("v") + .whereColumn("k") + .isEqualTo(literal(CqlVector.newInstance(0.1, 0.2)))) + .hasCql("DELETE v FROM foo WHERE k=[0.1, 0.2]"); + } + @Test public void should_generate_field_deletion() { assertThat( diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/insert/RegularInsertTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/insert/RegularInsertTest.java index 36133445b34..89c833ff1c6 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/insert/RegularInsertTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/insert/RegularInsertTest.java @@ -23,6 +23,7 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; import static org.assertj.core.api.Assertions.catchThrowable; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.querybuilder.term.Term; import com.datastax.oss.driver.internal.querybuilder.insert.DefaultInsert; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; @@ -41,6 +42,12 @@ public void should_generate_column_assignments() { .hasCql("INSERT INTO foo (a,b) VALUES (?,?)"); } + @Test + public void should_generate_vector_literals() { + assertThat(insertInto("foo").value("a", literal(CqlVector.newInstance(0.1, 0.2, 0.3)))) + .hasCql("INSERT INTO foo (a) VALUES ([0.1, 0.2, 0.3])"); + } + @Test public void should_keep_last_assignment_if_column_listed_twice() { assertThat( diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTableTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTableTest.java index 1567b0848cf..2c99b154b38 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTableTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTableTest.java @@ -108,4 +108,10 @@ public void should_generate_alter_table_with_no_compression() { assertThat(alterTable("bar").withNoCompression()) .hasCql("ALTER TABLE bar WITH compression={'sstable_compression':''}"); } + + @Test + public void should_generate_alter_table_with_vector() { + assertThat(alterTable("bar").alterColumn("v", DataTypes.vectorOf(DataTypes.FLOAT, 3))) + .hasCql("ALTER TABLE bar ALTER v TYPE vector"); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTypeTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTypeTest.java index 2becb9338f9..14bec0a6ce3 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTypeTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/AlterTypeTest.java @@ -53,4 +53,10 @@ public void should_generate_alter_table_with_rename_three_columns() { assertThat(alterType("bar").renameField("x", "y").renameField("u", "v").renameField("b", "a")) .hasCql("ALTER TYPE bar RENAME x TO y AND u TO v AND b TO a"); } + + @Test + public void should_generate_alter_type_with_vector() { + assertThat(alterType("foo", "bar").alterField("vec", DataTypes.vectorOf(DataTypes.FLOAT, 3))) + .hasCql("ALTER TYPE foo.bar ALTER vec TYPE vector"); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java index 7a5542c51f0..15cd12c75eb 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTableTest.java @@ -314,4 +314,13 @@ public void should_generate_create_table_time_window_compaction() { .hasCql( "CREATE TABLE bar (k int PRIMARY KEY,v text) WITH compaction={'class':'TimeWindowCompactionStrategy','compaction_window_size':10,'compaction_window_unit':'DAYS','timestamp_resolution':'MICROSECONDS','unsafe_aggressive_sstable_expiration':false}"); } + + @Test + public void should_generate_vector_column() { + assertThat( + createTable("foo") + .withPartitionKey("k", DataTypes.INT) + .withColumn("v", DataTypes.vectorOf(DataTypes.FLOAT, 3))) + .hasCql("CREATE TABLE foo (k int PRIMARY KEY,v vector)"); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTypeTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTypeTest.java index d881a0500cb..f7c15788a0f 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTypeTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/schema/CreateTypeTest.java @@ -83,4 +83,13 @@ public void should_create_type_with_collections() { .withField("map", DataTypes.mapOf(DataTypes.INT, DataTypes.TEXT))) .hasCql("CREATE TYPE ks1.type (map map)"); } + + @Test + public void should_create_type_with_vector() { + assertThat( + createType("ks1", "type") + .withField("c1", DataTypes.INT) + .withField("vec", DataTypes.vectorOf(DataTypes.FLOAT, 3))) + .hasCql("CREATE TYPE ks1.type (c1 int,vec vector)"); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectOrderingTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectOrderingTest.java index ff27fde4f8f..a9c618e9559 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectOrderingTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectOrderingTest.java @@ -23,6 +23,7 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.selectFrom; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.querybuilder.relation.Relation; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; import org.junit.Test; @@ -74,4 +75,23 @@ public void should_replace_previous_ordering() { .orderBy(ImmutableMap.of("c1", DESC, "c2", ASC))) .hasCql("SELECT * FROM foo WHERE k=1 ORDER BY c3 ASC,c1 DESC,c2 ASC"); } + + @Test + public void should_generate_ann_clause() { + assertThat( + selectFrom("foo") + .all() + .where(Relation.column("k").isEqualTo(literal(1))) + .orderByAnnOf("c1", CqlVector.newInstance(0.1, 0.2, 0.3))) + .hasCql("SELECT * FROM foo WHERE k=1 ORDER BY c1 ANN OF [0.1, 0.2, 0.3]"); + } + + @Test(expected = IllegalArgumentException.class) + public void should_fail_when_provided_ann_with_other_orderings() { + selectFrom("foo") + .all() + .where(Relation.column("k").isEqualTo(literal(1))) + .orderBy("c1", ASC) + .orderByAnnOf("c2", CqlVector.newInstance(0.1, 0.2, 0.3)); + } } diff --git a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectSelectorTest.java b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectSelectorTest.java index dc7cc98c6cc..7e03627d4b7 100644 --- a/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectSelectorTest.java +++ b/query-builder/src/test/java/com/datastax/oss/driver/api/querybuilder/select/SelectSelectorTest.java @@ -22,6 +22,7 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.raw; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.selectFrom; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.core.type.DataTypes; import com.datastax.oss.driver.api.core.type.codec.CodecNotFoundException; import com.datastax.oss.driver.api.querybuilder.CharsetCodec; @@ -230,6 +231,48 @@ public void should_generate_raw_selector() { .hasCql("SELECT bar,baz FROM foo"); } + @Test + public void should_generate_similarity_functions() { + Select similarity_cosine_clause = + selectFrom("cycling", "comments_vs") + .column("comment") + .function( + "similarity_cosine", + Selector.column("comment_vector"), + literal(CqlVector.newInstance(0.2, 0.15, 0.3, 0.2, 0.05))) + .orderByAnnOf("comment_vector", CqlVector.newInstance(0.1, 0.15, 0.3, 0.12, 0.05)) + .limit(1); + assertThat(similarity_cosine_clause) + .hasCql( + "SELECT comment,similarity_cosine(comment_vector,[0.2, 0.15, 0.3, 0.2, 0.05]) FROM cycling.comments_vs ORDER BY comment_vector ANN OF [0.1, 0.15, 0.3, 0.12, 0.05] LIMIT 1"); + + Select similarity_euclidean_clause = + selectFrom("cycling", "comments_vs") + .column("comment") + .function( + "similarity_euclidean", + Selector.column("comment_vector"), + literal(CqlVector.newInstance(0.2, 0.15, 0.3, 0.2, 0.05))) + .orderByAnnOf("comment_vector", CqlVector.newInstance(0.1, 0.15, 0.3, 0.12, 0.05)) + .limit(1); + assertThat(similarity_euclidean_clause) + .hasCql( + "SELECT comment,similarity_euclidean(comment_vector,[0.2, 0.15, 0.3, 0.2, 0.05]) FROM cycling.comments_vs ORDER BY comment_vector ANN OF [0.1, 0.15, 0.3, 0.12, 0.05] LIMIT 1"); + + Select similarity_dot_product_clause = + selectFrom("cycling", "comments_vs") + .column("comment") + .function( + "similarity_dot_product", + Selector.column("comment_vector"), + literal(CqlVector.newInstance(0.2, 0.15, 0.3, 0.2, 0.05))) + .orderByAnnOf("comment_vector", CqlVector.newInstance(0.1, 0.15, 0.3, 0.12, 0.05)) + .limit(1); + assertThat(similarity_dot_product_clause) + .hasCql( + "SELECT comment,similarity_dot_product(comment_vector,[0.2, 0.15, 0.3, 0.2, 0.05]) FROM cycling.comments_vs ORDER BY comment_vector ANN OF [0.1, 0.15, 0.3, 0.12, 0.05] LIMIT 1"); + } + @Test public void should_alias_selectors() { assertThat(selectFrom("foo").column("bar").as("baz")).hasCql("SELECT bar AS baz FROM foo"); From 8c1009959e988d9f7249151ad260f64aa6afed63 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 23 Dec 2024 08:20:42 +0100 Subject: [PATCH 32/48] Upgrade Guava to 33.3.1-jre patch by Lukasz Antoniak; reviewed by Alexandre Dutra and Bret McGuire for CASSJAVA-53 --- .../internal/core/cql/reactive/TestSubscriber.java | 5 +++-- guava-shaded/pom.xml | 10 ++-------- .../dse/driver/api/mapper/reactive/TestSubscriber.java | 6 +++++- pom.xml | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/core/src/test/java/com/datastax/dse/driver/internal/core/cql/reactive/TestSubscriber.java b/core/src/test/java/com/datastax/dse/driver/internal/core/cql/reactive/TestSubscriber.java index aed7a4dfc8e..652155e5309 100644 --- a/core/src/test/java/com/datastax/dse/driver/internal/core/cql/reactive/TestSubscriber.java +++ b/core/src/test/java/com/datastax/dse/driver/internal/core/cql/reactive/TestSubscriber.java @@ -81,7 +81,8 @@ public List getElements() { } public void awaitTermination() { - Uninterruptibles.awaitUninterruptibly(latch, 1, TimeUnit.MINUTES); - if (latch.getCount() > 0) fail("subscriber not terminated"); + if (!Uninterruptibles.awaitUninterruptibly(latch, 1, TimeUnit.MINUTES)) { + fail("subscriber not terminated"); + } } } diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index 9854fcc48ba..f480f9258cc 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -45,14 +45,6 @@ com.google.errorprone error_prone_annotations - - com.google.j2objc - j2objc-annotations - - - org.codehaus.mojo - animal-sniffer-annotations - @@ -78,6 +70,8 @@ org.apache.cassandra:java-driver-guava-shaded com.google.guava:guava + com.google.guava:failureaccess + com.google.j2objc:j2objc-annotations diff --git a/mapper-runtime/src/test/java/com/datastax/dse/driver/api/mapper/reactive/TestSubscriber.java b/mapper-runtime/src/test/java/com/datastax/dse/driver/api/mapper/reactive/TestSubscriber.java index 6f23cfca98a..6886b9a7622 100644 --- a/mapper-runtime/src/test/java/com/datastax/dse/driver/api/mapper/reactive/TestSubscriber.java +++ b/mapper-runtime/src/test/java/com/datastax/dse/driver/api/mapper/reactive/TestSubscriber.java @@ -17,6 +17,8 @@ */ package com.datastax.dse.driver.api.mapper.reactive; +import static org.assertj.core.api.Fail.fail; + import com.datastax.oss.driver.shaded.guava.common.util.concurrent.Uninterruptibles; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; @@ -70,6 +72,8 @@ public List getElements() { } public void awaitTermination() { - Uninterruptibles.awaitUninterruptibly(latch, 1, TimeUnit.MINUTES); + if (!Uninterruptibles.awaitUninterruptibly(latch, 1, TimeUnit.MINUTES)) { + fail("subscriber not terminated"); + } } } diff --git a/pom.xml b/pom.xml index 620cf1db4bb..c61e6485fd3 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,7 @@ com.google.guava guava - 25.1-jre + 33.3.1-jre com.typesafe From 75a269d04d49630032be6afedae2ded0c4334e42 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 30 Dec 2024 07:18:06 +0100 Subject: [PATCH 33/48] Do not always cleanup Guava shaded module before packaging --- guava-shaded/pom.xml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index f480f9258cc..6a22663e956 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -94,21 +94,6 @@ - - maven-clean-plugin - - - clean-classes - package - - clean - - - ${project.build.outputDirectory} - - - - maven-dependency-plugin From 01671d99247bdc783c832c128a8570ba846875c4 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Fri, 10 Jan 2025 16:05:42 +0100 Subject: [PATCH 34/48] Revert "Do not always cleanup Guava shaded module before packaging" This reverts commit 5be52ec1a8d014c81566180c731b828a591082da. --- guava-shaded/pom.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index 6a22663e956..f480f9258cc 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -94,6 +94,21 @@ + + maven-clean-plugin + + + clean-classes + package + + clean + + + ${project.build.outputDirectory} + + + + maven-dependency-plugin From 342e2dcf47afab238f357ac2afde65b079ce6b79 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Fri, 10 Jan 2025 16:16:26 +0100 Subject: [PATCH 35/48] Conditionally compile shaded Guava module --- guava-shaded/pom.xml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index f480f9258cc..ca8f0161b04 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -56,6 +56,32 @@ + + + org.codehaus.mojo + build-helper-maven-plugin + 1.12 + + + regex-property + + regex-property + + + maven.main.skip + ${java.version} + ^(?!1.8).+ + true + false + + + + maven-shade-plugin @@ -95,6 +121,12 @@ + maven-clean-plugin From 2e0c44c020819c928730e1aac812bf2f475d8256 Mon Sep 17 00:00:00 2001 From: SiyaoIsHiding <113857408+SiyaoIsHiding@users.noreply.github.com> Date: Sun, 26 Jan 2025 22:38:52 +0800 Subject: [PATCH 36/48] JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit patch by Jane He; reviewed by Bret McGuire and João Reis reference: https://github.com/apache/cassandra-java-driver/pull/1952 --- core/revapi.json | 297 ++++++++++++++++++ .../oss/driver/api/core/data/CqlVector.java | 72 ++++- .../driver/api/core/data/GettableById.java | 2 +- .../driver/api/core/data/GettableByIndex.java | 3 +- .../driver/api/core/data/GettableByName.java | 2 +- .../driver/api/core/data/SettableById.java | 2 +- .../driver/api/core/data/SettableByIndex.java | 2 +- .../driver/api/core/data/SettableByName.java | 2 +- .../oss/driver/api/core/type/DataTypes.java | 19 +- .../driver/api/core/type/codec/TypeCodec.java | 6 + .../api/core/type/codec/TypeCodecs.java | 4 +- .../api/core/type/reflect/GenericType.java | 6 +- .../parsing/DataTypeClassNameParser.java | 8 + .../internal/core/type/DefaultVectorType.java | 2 +- .../internal/core/type/codec/BigIntCodec.java | 7 + .../core/type/codec/BooleanCodec.java | 7 + .../internal/core/type/codec/DoubleCodec.java | 7 + .../internal/core/type/codec/FloatCodec.java | 7 + .../internal/core/type/codec/IntCodec.java | 7 + .../core/type/codec/TimestampCodec.java | 7 + .../internal/core/type/codec/UuidCodec.java | 7 + .../internal/core/type/codec/VectorCodec.java | 86 +++-- .../extras/time/TimestampMillisCodec.java | 7 + .../codec/registry/CachingCodecRegistry.java | 12 +- .../internal/core/type/util/VIntCoding.java | 77 ++++- .../driver/api/core/data/CqlVectorTest.java | 167 +++++----- .../core/type/codec/VectorCodecTest.java | 265 ++++++++++++---- ...CachingCodecRegistryTestDataProviders.java | 20 ++ .../core/type/util/VIntCodingTest.java | 86 +++++ .../oss/driver/core/data/DataTypeIT.java | 38 ++- 30 files changed, 1002 insertions(+), 232 deletions(-) create mode 100644 core/src/test/java/com/datastax/oss/driver/internal/core/type/util/VIntCodingTest.java diff --git a/core/revapi.json b/core/revapi.json index 1c875895d6c..5aa46a3ccad 100644 --- a/core/revapi.json +++ b/core/revapi.json @@ -7089,6 +7089,303 @@ "new": "method com.datastax.oss.driver.api.core.cql.SimpleStatement com.datastax.oss.driver.api.core.cql.SimpleStatement::setNamedValues(java.util.Map)", "annotation": "@edu.umd.cs.findbugs.annotations.CheckReturnValue", "justification": "Annotate mutating methods with @CheckReturnValue" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::from(java.lang.String, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeChanged", + "old": "method T com.datastax.oss.driver.api.core.data.CqlVector::get(int)", + "new": "method T com.datastax.oss.driver.api.core.data.CqlVector::get(int)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method java.util.Iterator com.datastax.oss.driver.api.core.data.CqlVector::iterator()", + "new": "method java.util.Iterator com.datastax.oss.driver.api.core.data.CqlVector::iterator()", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(===V[]===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(===V[]===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(V[])", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(V[])", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(V[])", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(V[])", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(===java.util.List===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(===java.util.List===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(java.util.List)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(java.util.List)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(java.util.List)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::newInstance(java.util.List)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeChanged", + "old": "parameter T com.datastax.oss.driver.api.core.data.CqlVector::set(int, ===T===)", + "new": "parameter T com.datastax.oss.driver.api.core.data.CqlVector::set(int, ===T===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeChanged", + "old": "method T com.datastax.oss.driver.api.core.data.CqlVector::set(int, T)", + "new": "method T com.datastax.oss.driver.api.core.data.CqlVector::set(int, T)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method java.util.Spliterator java.lang.Iterable::spliterator() @ com.datastax.oss.driver.api.core.data.CqlVector", + "new": "method java.util.Spliterator java.lang.Iterable::spliterator() @ com.datastax.oss.driver.api.core.data.CqlVector", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method java.util.stream.Stream com.datastax.oss.driver.api.core.data.CqlVector::stream()", + "new": "method java.util.stream.Stream com.datastax.oss.driver.api.core.data.CqlVector::stream()", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::subVector(int, int)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.CqlVector::subVector(int, int)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.class.noLongerImplementsInterface", + "old": "class com.datastax.oss.driver.api.core.data.CqlVector", + "new": "class com.datastax.oss.driver.api.core.data.CqlVector", + "interface": "java.lang.Iterable", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "class com.datastax.oss.driver.api.core.data.CqlVector", + "new": "class com.datastax.oss.driver.api.core.data.CqlVector", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.class.superTypeTypeParametersChanged", + "old": "class com.datastax.oss.driver.api.core.data.CqlVector", + "new": "class com.datastax.oss.driver.api.core.data.CqlVector", + "oldSuperType": "java.lang.Iterable", + "newSuperType": "java.lang.Iterable", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, ===java.lang.Class===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableById::getVector(com.datastax.oss.driver.api.core.CqlIdentifier, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, ===java.lang.Class===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByIndex::getVector(int, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, ===java.lang.Class===)", + "new": "parameter com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.data.CqlVector com.datastax.oss.driver.api.core.data.GettableByName::getVector(java.lang.String, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "new": "method SelfT com.datastax.oss.driver.api.core.data.SettableById>::setVector(com.datastax.oss.driver.api.core.CqlIdentifier, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "new": "method SelfT com.datastax.oss.driver.api.core.data.SettableByIndex>::setVector(int, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, ===com.datastax.oss.driver.api.core.data.CqlVector===, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "new": "parameter SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector, ===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "new": "method SelfT com.datastax.oss.driver.api.core.data.SettableByName>::setVector(java.lang.String, com.datastax.oss.driver.api.core.data.CqlVector, java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "new": "parameter com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(com.datastax.oss.driver.api.core.type.VectorType, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "new": "parameter com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, ===com.datastax.oss.driver.api.core.type.codec.TypeCodec===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "new": "method com.datastax.oss.driver.api.core.type.codec.TypeCodec> com.datastax.oss.driver.api.core.type.codec.TypeCodecs::vectorOf(int, com.datastax.oss.driver.api.core.type.codec.TypeCodec)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(===com.datastax.oss.driver.api.core.type.reflect.GenericType===)", + "new": "parameter com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(===com.datastax.oss.driver.api.core.type.reflect.GenericType===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(com.datastax.oss.driver.api.core.type.reflect.GenericType)", + "new": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(com.datastax.oss.driver.api.core.type.reflect.GenericType)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(com.datastax.oss.driver.api.core.type.reflect.GenericType)", + "new": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(com.datastax.oss.driver.api.core.type.reflect.GenericType)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.parameterTypeParameterChanged", + "old": "parameter com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(===java.lang.Class===)", + "new": "parameter com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(===java.lang.Class===)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.method.returnTypeTypeParametersChanged", + "old": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" + }, + { + "code": "java.generics.formalTypeParameterChanged", + "old": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", + "new": "method com.datastax.oss.driver.api.core.type.reflect.GenericType> com.datastax.oss.driver.api.core.type.reflect.GenericType::vectorOf(java.lang.Class)", + "justification": "JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0)" } ] } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/CqlVector.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/CqlVector.java index 911b6187f6d..8089d551750 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/CqlVector.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/CqlVector.java @@ -18,11 +18,10 @@ package com.datastax.oss.driver.api.core.data; import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.internal.core.type.codec.ParseUtils; import com.datastax.oss.driver.shaded.guava.common.base.Preconditions; import com.datastax.oss.driver.shaded.guava.common.base.Predicates; -import com.datastax.oss.driver.shaded.guava.common.base.Splitter; import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; -import com.datastax.oss.driver.shaded.guava.common.collect.Streams; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.io.InvalidObjectException; @@ -35,7 +34,6 @@ import java.util.Iterator; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -52,7 +50,7 @@ * where possible we've tried to make the API of this class similar to the equivalent methods on * {@link List}. */ -public class CqlVector implements Iterable, Serializable { +public class CqlVector implements Iterable, Serializable { /** * Create a new CqlVector containing the specified values. @@ -60,7 +58,7 @@ public class CqlVector implements Iterable, Serializable { * @param vals the collection of values to wrap. * @return a CqlVector wrapping those values */ - public static CqlVector newInstance(V... vals) { + public static CqlVector newInstance(V... vals) { // Note that Array.asList() guarantees the return of an array which implements RandomAccess return new CqlVector(Arrays.asList(vals)); @@ -73,29 +71,64 @@ public static CqlVector newInstance(V... vals) { * @param list the collection of values to wrap. * @return a CqlVector wrapping those values */ - public static CqlVector newInstance(List list) { + public static CqlVector newInstance(List list) { Preconditions.checkArgument(list != null, "Input list should not be null"); return new CqlVector(list); } /** - * Create a new CqlVector instance from the specified string representation. Note that this method - * is intended to mirror {@link #toString()}; passing this method the output from a toString - * call on some CqlVector should return a CqlVector that is equal to the origin instance. + * Create a new CqlVector instance from the specified string representation. * * @param str a String representation of a CqlVector * @param subtypeCodec * @return a new CqlVector built from the String representation */ - public static CqlVector from( - @NonNull String str, @NonNull TypeCodec subtypeCodec) { + public static CqlVector from(@NonNull String str, @NonNull TypeCodec subtypeCodec) { Preconditions.checkArgument(str != null, "Cannot create CqlVector from null string"); Preconditions.checkArgument(!str.isEmpty(), "Cannot create CqlVector from empty string"); - ArrayList vals = - Streams.stream(Splitter.on(", ").split(str.substring(1, str.length() - 1))) - .map(subtypeCodec::parse) - .collect(Collectors.toCollection(ArrayList::new)); - return new CqlVector(vals); + if (str.equalsIgnoreCase("NULL")) return null; + + int idx = ParseUtils.skipSpaces(str, 0); + if (str.charAt(idx++) != '[') + throw new IllegalArgumentException( + String.format( + "Cannot parse vector value from \"%s\", at character %d expecting '[' but got '%c'", + str, idx, str.charAt(idx))); + + idx = ParseUtils.skipSpaces(str, idx); + + if (str.charAt(idx) == ']') { + return new CqlVector<>(new ArrayList<>()); + } + + List list = new ArrayList<>(); + while (idx < str.length()) { + int n; + try { + n = ParseUtils.skipCQLValue(str, idx); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + String.format( + "Cannot parse vector value from \"%s\", invalid CQL value at character %d", + str, idx), + e); + } + + list.add(subtypeCodec.parse(str.substring(idx, n))); + idx = n; + + idx = ParseUtils.skipSpaces(str, idx); + if (str.charAt(idx) == ']') return new CqlVector<>(list); + if (str.charAt(idx++) != ',') + throw new IllegalArgumentException( + String.format( + "Cannot parse vector value from \"%s\", at character %d expecting ',' but got '%c'", + str, idx, str.charAt(idx))); + + idx = ParseUtils.skipSpaces(str, idx); + } + throw new IllegalArgumentException( + String.format("Malformed vector value \"%s\", missing closing ']'", str)); } private final List list; @@ -194,6 +227,11 @@ public int hashCode() { return Objects.hash(list); } + /** + * The string representation of the vector. Elements, like strings, may not be properly quoted. + * + * @return the string representation + */ @Override public String toString() { return Iterables.toString(this.list); @@ -205,7 +243,7 @@ public String toString() { * * @param inner type of CqlVector, assume Number is always Serializable. */ - private static class SerializationProxy implements Serializable { + private static class SerializationProxy implements Serializable { private static final long serialVersionUID = 1; diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableById.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableById.java index 0a24214b20a..8393bc9f758 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableById.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableById.java @@ -531,7 +531,7 @@ default CqlDuration getCqlDuration(@NonNull CqlIdentifier id) { * @throws IllegalArgumentException if the id is invalid. */ @Nullable - default CqlVector getVector( + default CqlVector getVector( @NonNull CqlIdentifier id, @NonNull Class elementsClass) { return getVector(firstIndexOf(id), elementsClass); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByIndex.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByIndex.java index 53541b0ac58..bb75bd9a2b4 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByIndex.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByIndex.java @@ -446,8 +446,7 @@ default CqlDuration getCqlDuration(int i) { * @throws IndexOutOfBoundsException if the index is invalid. */ @Nullable - default CqlVector getVector( - int i, @NonNull Class elementsClass) { + default CqlVector getVector(int i, @NonNull Class elementsClass) { return get(i, GenericType.vectorOf(elementsClass)); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByName.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByName.java index ec3ee362ca8..b0a4660033b 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByName.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/GettableByName.java @@ -527,7 +527,7 @@ default CqlDuration getCqlDuration(@NonNull String name) { * @throws IllegalArgumentException if the name is invalid. */ @Nullable - default CqlVector getVector( + default CqlVector getVector( @NonNull String name, @NonNull Class elementsClass) { return getVector(firstIndexOf(name), elementsClass); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableById.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableById.java index 8452446205e..0f5e3cd9daa 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableById.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableById.java @@ -573,7 +573,7 @@ default SelfT setCqlDuration(@NonNull CqlIdentifier id, @Nullable CqlDuration v) */ @NonNull @CheckReturnValue - default SelfT setVector( + default SelfT setVector( @NonNull CqlIdentifier id, @Nullable CqlVector v, @NonNull Class elementsClass) { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByIndex.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByIndex.java index bb55db3adde..4ecdf647590 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByIndex.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByIndex.java @@ -425,7 +425,7 @@ default SelfT setCqlDuration(int i, @Nullable CqlDuration v) { */ @NonNull @CheckReturnValue - default SelfT setVector( + default SelfT setVector( int i, @Nullable CqlVector v, @NonNull Class elementsClass) { return set(i, v, GenericType.vectorOf(elementsClass)); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByName.java b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByName.java index c25a7074373..afe9ba59f64 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByName.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/data/SettableByName.java @@ -572,7 +572,7 @@ default SelfT setCqlDuration(@NonNull String name, @Nullable CqlDuration v) { */ @NonNull @CheckReturnValue - default SelfT setVector( + default SelfT setVector( @NonNull String name, @Nullable CqlVector v, @NonNull Class elementsClass) { diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/type/DataTypes.java b/core/src/main/java/com/datastax/oss/driver/api/core/type/DataTypes.java index 3a341eaa5aa..492fc121c71 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/type/DataTypes.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/type/DataTypes.java @@ -27,12 +27,10 @@ import com.datastax.oss.driver.internal.core.type.DefaultTupleType; import com.datastax.oss.driver.internal.core.type.DefaultVectorType; import com.datastax.oss.driver.internal.core.type.PrimitiveType; -import com.datastax.oss.driver.shaded.guava.common.base.Splitter; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.protocol.internal.ProtocolConstants; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.Arrays; -import java.util.List; /** Constants and factory methods to obtain data type instances. */ public class DataTypes { @@ -59,7 +57,6 @@ public class DataTypes { public static final DataType DURATION = new PrimitiveType(ProtocolConstants.DataType.DURATION); private static final DataTypeClassNameParser classNameParser = new DataTypeClassNameParser(); - private static final Splitter paramSplitter = Splitter.on(',').trimResults(); @NonNull public static DataType custom(@NonNull String className) { @@ -68,20 +65,8 @@ public static DataType custom(@NonNull String className) { if (className.equals("org.apache.cassandra.db.marshal.DurationType")) return DURATION; /* Vector support is currently implemented as a custom type but is also parameterized */ - if (className.startsWith(DefaultVectorType.VECTOR_CLASS_NAME)) { - List params = - paramSplitter.splitToList( - className.substring( - DefaultVectorType.VECTOR_CLASS_NAME.length() + 1, className.length() - 1)); - DataType subType = classNameParser.parse(params.get(0), AttachmentPoint.NONE); - int dimensions = Integer.parseInt(params.get(1)); - if (dimensions <= 0) { - throw new IllegalArgumentException( - String.format( - "Request to create vector of size %d, size must be positive", dimensions)); - } - return new DefaultVectorType(subType, dimensions); - } + if (className.startsWith(DefaultVectorType.VECTOR_CLASS_NAME)) + return classNameParser.parse(className, AttachmentPoint.NONE); return new DefaultCustomType(className); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodec.java b/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodec.java index 05ae3980823..d6afbe0380a 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodec.java @@ -28,6 +28,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; /** * Manages the two-way conversion between a CQL type and a Java type. @@ -234,4 +235,9 @@ default boolean accepts(@NonNull DataType cqlType) { */ @Nullable JavaTypeT parse(@Nullable String value); + + @NonNull + default Optional serializedSize() { + return Optional.empty(); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodecs.java b/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodecs.java index 9f2fd5cc69e..68f1b07b106 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodecs.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/type/codec/TypeCodecs.java @@ -210,13 +210,13 @@ public static TypeCodec tupleOf(@NonNull TupleType cqlType) { return new TupleCodec(cqlType); } - public static TypeCodec> vectorOf( + public static TypeCodec> vectorOf( @NonNull VectorType type, @NonNull TypeCodec subtypeCodec) { return new VectorCodec( DataTypes.vectorOf(subtypeCodec.getCqlType(), type.getDimensions()), subtypeCodec); } - public static TypeCodec> vectorOf( + public static TypeCodec> vectorOf( int dimensions, @NonNull TypeCodec subtypeCodec) { return new VectorCodec(DataTypes.vectorOf(subtypeCodec.getCqlType(), dimensions), subtypeCodec); } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/type/reflect/GenericType.java b/core/src/main/java/com/datastax/oss/driver/api/core/type/reflect/GenericType.java index c6482d4f4d1..d22b6f1bfaf 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/type/reflect/GenericType.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/type/reflect/GenericType.java @@ -151,8 +151,7 @@ public static GenericType> setOf(@NonNull GenericType elementType) } @NonNull - public static GenericType> vectorOf( - @NonNull Class elementType) { + public static GenericType> vectorOf(@NonNull Class elementType) { TypeToken> token = new TypeToken>() {}.where( new TypeParameter() {}, TypeToken.of(elementType)); @@ -160,8 +159,7 @@ public static GenericType> vectorOf( } @NonNull - public static GenericType> vectorOf( - @NonNull GenericType elementType) { + public static GenericType> vectorOf(@NonNull GenericType elementType) { TypeToken> token = new TypeToken>() {}.where(new TypeParameter() {}, elementType.token); return new GenericType<>(token); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/schema/parsing/DataTypeClassNameParser.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/schema/parsing/DataTypeClassNameParser.java index fd6f1a4bd51..bf252d0bc57 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/schema/parsing/DataTypeClassNameParser.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/schema/parsing/DataTypeClassNameParser.java @@ -34,6 +34,7 @@ import com.datastax.oss.protocol.internal.util.Bytes; import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -164,6 +165,13 @@ private DataType parse( return new DefaultTupleType(componentTypesBuilder.build(), attachmentPoint); } + if (next.startsWith("org.apache.cassandra.db.marshal.VectorType")) { + Iterator rawTypes = parser.getTypeParameters().iterator(); + DataType subtype = parse(rawTypes.next(), userTypes, attachmentPoint, logPrefix); + int dimensions = Integer.parseInt(rawTypes.next()); + return DataTypes.vectorOf(subtype, dimensions); + } + DataType type = NATIVE_TYPES_BY_CLASS_NAME.get(next); return type == null ? DataTypes.custom(toParse) : type; } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java index c9180d44edc..0b1ced94769 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/DefaultVectorType.java @@ -78,7 +78,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(super.hashCode(), subtype, dimensions); + return Objects.hash(DefaultVectorType.class, subtype, dimensions); } @Override diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BigIntCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BigIntCodec.java index 2b3b8255cc1..8496da17fa6 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BigIntCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BigIntCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -90,4 +91,10 @@ public Long parse(@Nullable String value) { String.format("Cannot parse 64-bits long value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(8); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BooleanCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BooleanCodec.java index 7a982a9e6ca..af388982be9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BooleanCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/BooleanCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -98,4 +99,10 @@ public Boolean parse(@Nullable String value) { String.format("Cannot parse boolean value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(1); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/DoubleCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/DoubleCodec.java index 28eff6f9463..b01847517d9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/DoubleCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/DoubleCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -90,4 +91,10 @@ public Double parse(@Nullable String value) { String.format("Cannot parse 64-bits double value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(8); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/FloatCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/FloatCodec.java index 11786dbc77d..fd851edfad3 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/FloatCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/FloatCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -90,4 +91,10 @@ public Float parse(@Nullable String value) { String.format("Cannot parse 32-bits float value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(4); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/IntCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/IntCodec.java index e5bb530ba79..b11b164a445 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/IntCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/IntCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import net.jcip.annotations.ThreadSafe; @ThreadSafe @@ -90,4 +91,10 @@ public Integer parse(@Nullable String value) { String.format("Cannot parse 32-bits int value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(4); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/TimestampCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/TimestampCodec.java index eeba3c7c66c..964f774c8d9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/TimestampCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/TimestampCodec.java @@ -33,6 +33,7 @@ import java.time.Instant; import java.time.ZoneId; import java.util.Date; +import java.util.Optional; import java.util.TimeZone; import net.jcip.annotations.ThreadSafe; @@ -293,4 +294,10 @@ public Instant parse(@Nullable String value) { String.format("Cannot parse timestamp value from \"%s\"", value)); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(8); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UuidCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UuidCodec.java index 57feac4ae7e..cc5f48dbe52 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UuidCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/UuidCodec.java @@ -25,6 +25,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; +import java.util.Optional; import java.util.UUID; import net.jcip.annotations.ThreadSafe; @@ -95,4 +96,10 @@ public UUID parse(@Nullable String value) { String.format("Cannot parse UUID value from \"%s\"", value), e); } } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(16); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodec.java index 2c4d2200b13..1f8ce1a7166 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodec.java @@ -24,7 +24,7 @@ import com.datastax.oss.driver.api.core.type.codec.TypeCodec; import com.datastax.oss.driver.api.core.type.reflect.GenericType; import com.datastax.oss.driver.internal.core.type.DefaultVectorType; -import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; +import com.datastax.oss.driver.internal.core.type.util.VIntCoding; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import java.nio.ByteBuffer; @@ -32,8 +32,10 @@ import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.stream.Collectors; -public class VectorCodec implements TypeCodec> { +public class VectorCodec implements TypeCodec> { private final VectorType cqlType; private final GenericType> javaType; @@ -55,6 +57,14 @@ public GenericType> getJavaType() { return this.javaType; } + @NonNull + @Override + public Optional serializedSize() { + return subtypeCodec.serializedSize().isPresent() + ? Optional.of(subtypeCodec.serializedSize().get() * cqlType.getDimensions()) + : Optional.empty(); + } + @NonNull @Override public DataType getCqlType() { @@ -65,6 +75,7 @@ public DataType getCqlType() { @Override public ByteBuffer encode( @Nullable CqlVector value, @NonNull ProtocolVersion protocolVersion) { + boolean isVarSized = !subtypeCodec.serializedSize().isPresent(); if (value == null || cqlType.getDimensions() <= 0) { return null; } @@ -92,14 +103,28 @@ public ByteBuffer encode( if (valueBuff == null) { throw new NullPointerException("Vector elements cannot encode to CQL NULL"); } - allValueBuffsSize += valueBuff.limit(); + int elementSize = valueBuff.limit(); + if (isVarSized) { + allValueBuffsSize += VIntCoding.computeVIntSize(elementSize); + } + allValueBuffsSize += elementSize; valueBuff.rewind(); valueBuffs[i] = valueBuff; } + // if too many elements, throw + if (values.hasNext()) { + throw new IllegalArgumentException( + String.format( + "Too many elements; must provide elements for %d dimensions", + cqlType.getDimensions())); + } /* Since we already did an early return for <= 0 dimensions above */ assert valueBuffs.length > 0; ByteBuffer rv = ByteBuffer.allocate(allValueBuffsSize); for (int i = 0; i < cqlType.getDimensions(); ++i) { + if (isVarSized) { + VIntCoding.writeUnsignedVInt32(valueBuffs[i].remaining(), rv); + } rv.put(valueBuffs[i]); } rv.flip(); @@ -114,39 +139,58 @@ public CqlVector decode( return null; } - /* Determine element size by dividing count of remaining bytes by number of elements. This should have a remainder - of zero if we assume all elements are of uniform size (which is really a terrible assumption). - - TODO: We should probably tweak serialization format for vectors if we're going to allow them for arbitrary subtypes. - Elements should at least precede themselves with their size (along the lines of what lists do). */ - int elementSize = Math.floorDiv(bytes.remaining(), cqlType.getDimensions()); - if (!(bytes.remaining() % cqlType.getDimensions() == 0)) { - throw new IllegalArgumentException( - String.format( - "Expected elements of uniform size, observed %d elements with total bytes %d", - cqlType.getDimensions(), bytes.remaining())); - } - + // Upfront check for fixed-size types only + subtypeCodec + .serializedSize() + .ifPresent( + (fixed_size) -> { + if (bytes.remaining() != cqlType.getDimensions() * fixed_size) { + throw new IllegalArgumentException( + String.format( + "Expected elements of uniform size, observed %d elements with total bytes %d", + cqlType.getDimensions(), bytes.remaining())); + } + }); + ; ByteBuffer slice = bytes.slice(); List rv = new ArrayList(cqlType.getDimensions()); for (int i = 0; i < cqlType.getDimensions(); ++i) { - // Set the limit for the current element + + int size = + subtypeCodec + .serializedSize() + .orElseGet(() -> VIntCoding.getUnsignedVInt32(slice, slice.position())); + // If we aren't dealing with a fixed-size type we need to move the current slice position + // beyond the vint-encoded size of the current element. Ideally this would be + // serializedSize().ifNotPresent(Consumer) but the Optional API isn't doing us any favors + // there. + if (!subtypeCodec.serializedSize().isPresent()) + slice.position(slice.position() + VIntCoding.computeUnsignedVIntSize(size)); int originalPosition = slice.position(); - slice.limit(originalPosition + elementSize); + slice.limit(originalPosition + size); rv.add(this.subtypeCodec.decode(slice, protocolVersion)); // Move to the start of the next element - slice.position(originalPosition + elementSize); + slice.position(originalPosition + size); // Reset the limit to the end of the buffer slice.limit(slice.capacity()); } + // if too many elements, throw + if (slice.hasRemaining()) { + throw new IllegalArgumentException( + String.format( + "Too many elements; must provide elements for %d dimensions", + cqlType.getDimensions())); + } + return CqlVector.newInstance(rv); } @NonNull @Override - public String format(@Nullable CqlVector value) { - return value == null ? "NULL" : Iterables.toString(value); + public String format(CqlVector value) { + if (value == null) return "NULL"; + return value.stream().map(subtypeCodec::format).collect(Collectors.joining(", ", "[", "]")); } @Nullable diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/extras/time/TimestampMillisCodec.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/extras/time/TimestampMillisCodec.java index a15495a432d..12e3e839d2a 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/extras/time/TimestampMillisCodec.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/extras/time/TimestampMillisCodec.java @@ -31,6 +31,7 @@ import java.time.Instant; import java.time.ZoneId; import java.util.Objects; +import java.util.Optional; import net.jcip.annotations.Immutable; /** @@ -114,4 +115,10 @@ public String format(@Nullable Long value) { Instant instant = value == null ? null : Instant.ofEpochMilli(value); return timestampCodec.format(instant); } + + @NonNull + @Override + public Optional serializedSize() { + return Optional.of(8); + } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistry.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistry.java index 495d6227d93..3af5a30ba27 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistry.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistry.java @@ -394,10 +394,9 @@ protected GenericType inspectType(@NonNull Object value, @Nullable DataType c "Can't infer vector codec because the first element is null " + "(note that CQL does not allow null values in collections)"); } - GenericType elementType = - (GenericType) - inspectType( - firstElement, cqlType == null ? null : ((VectorType) cqlType).getElementType()); + GenericType elementType = + inspectType( + firstElement, cqlType == null ? null : ((VectorType) cqlType).getElementType()); return GenericType.vectorOf(elementType); } } else { @@ -421,8 +420,7 @@ protected GenericType inferJavaTypeFromCqlType(@NonNull DataType cqlType) { inferJavaTypeFromCqlType(keyType), inferJavaTypeFromCqlType(valueType)); } else if (cqlType instanceof VectorType) { DataType elementType = ((VectorType) cqlType).getElementType(); - GenericType numberType = - (GenericType) inferJavaTypeFromCqlType(elementType); + GenericType numberType = inferJavaTypeFromCqlType(elementType); return GenericType.vectorOf(numberType); } switch (cqlType.getProtocolCode()) { @@ -657,7 +655,7 @@ protected TypeCodec createCodec( /* For a vector type we'll always get back an instance of TypeCodec due to the * type of CqlVector... but getElementCodecForCqlAndJavaType() is a generalized function that can't * return this more precise type. Thus the cast here. */ - TypeCodec elementCodec = + TypeCodec elementCodec = uncheckedCast(getElementCodecForCqlAndJavaType(vectorType, token, isJavaCovariant)); return TypeCodecs.vectorOf(vectorType, elementCodec); } else if (cqlType instanceof CustomType diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/type/util/VIntCoding.java b/core/src/main/java/com/datastax/oss/driver/internal/core/type/util/VIntCoding.java index 5ee375a81e5..552f84f2ae1 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/type/util/VIntCoding.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/type/util/VIntCoding.java @@ -49,6 +49,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.nio.ByteBuffer; /** * Variable length encoding inspired from Google > 6; } + + public static void writeUnsignedVInt32(int value, ByteBuffer output) { + writeUnsignedVInt((long) value, output); + } + + public static void writeUnsignedVInt(long value, ByteBuffer output) { + int size = VIntCoding.computeUnsignedVIntSize(value); + if (size == 1) { + output.put((byte) value); + return; + } + + output.put(VIntCoding.encodeVInt(value, size), 0, size); + } + + /** + * Read up to a 32-bit integer back, using the unsigned (no zigzag) encoding. + * + *

Note this method is the same as {@link #readUnsignedVInt(DataInput)}, except that we do + * *not* block if there are not enough bytes in the buffer to reconstruct the value. + * + * @throws VIntOutOfRangeException If the vint doesn't fit into a 32-bit integer + */ + public static int getUnsignedVInt32(ByteBuffer input, int readerIndex) { + return checkedCast(getUnsignedVInt(input, readerIndex)); + } + + public static long getUnsignedVInt(ByteBuffer input, int readerIndex) { + return getUnsignedVInt(input, readerIndex, input.limit()); + } + + public static long getUnsignedVInt(ByteBuffer input, int readerIndex, int readerLimit) { + if (readerIndex < 0) + throw new IllegalArgumentException( + "Reader index should be non-negative, but was " + readerIndex); + + if (readerIndex >= readerLimit) return -1; + + int firstByte = input.get(readerIndex++); + + // Bail out early if this is one byte, necessary or it fails later + if (firstByte >= 0) return firstByte; + + int size = numberOfExtraBytesToRead(firstByte); + if (readerIndex + size > readerLimit) return -1; + + long retval = firstByte & firstByteValueMask(size); + for (int ii = 0; ii < size; ii++) { + byte b = input.get(readerIndex++); + retval <<= 8; + retval |= b & 0xff; + } + + return retval; + } + + public static int checkedCast(long value) { + int result = (int) value; + if ((long) result != value) throw new VIntOutOfRangeException(value); + return result; + } + + /** + * Throw when attempting to decode a vint and the output type doesn't have enough space to fit the + * value that was decoded + */ + public static class VIntOutOfRangeException extends RuntimeException { + public final long value; + + private VIntOutOfRangeException(long value) { + super(value + " is out of range for a 32-bit integer"); + this.value = value; + } + } } diff --git a/core/src/test/java/com/datastax/oss/driver/api/core/data/CqlVectorTest.java b/core/src/test/java/com/datastax/oss/driver/api/core/data/CqlVectorTest.java index 90f4cc6e776..3e0872cb946 100644 --- a/core/src/test/java/com/datastax/oss/driver/api/core/data/CqlVectorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/api/core/data/CqlVectorTest.java @@ -23,56 +23,60 @@ import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; import com.datastax.oss.driver.internal.SerializationHelper; -import com.datastax.oss.driver.shaded.guava.common.collect.Iterators; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; import java.io.ByteArrayInputStream; import java.io.ObjectInputStream; import java.io.ObjectStreamException; +import java.time.LocalTime; import java.util.AbstractList; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; import org.assertj.core.util.Lists; import org.junit.Test; +import org.junit.runner.RunWith; +@RunWith(DataProviderRunner.class) public class CqlVectorTest { - private static final Float[] VECTOR_ARGS = {1.0f, 2.5f}; - - private void validate_built_vector(CqlVector vec) { + @DataProvider + public static Object[][] dataProvider() { + return new Object[][] { + {new Float[] {1.0f, 2.5f}}, + {new LocalTime[] {LocalTime.of(1, 2), LocalTime.of(3, 4)}}, + {new List[] {Arrays.asList(1, 2), Arrays.asList(3, 4)}}, + {new CqlVector[] {CqlVector.newInstance("a", "bc"), CqlVector.newInstance("d", "ef")}} + }; + } + private void validate_built_vector(CqlVector vec, Object[] expectedVals) { assertThat(vec.size()).isEqualTo(2); assertThat(vec.isEmpty()).isFalse(); - assertThat(vec.get(0)).isEqualTo(VECTOR_ARGS[0]); - assertThat(vec.get(1)).isEqualTo(VECTOR_ARGS[1]); + assertThat(vec.get(0)).isEqualTo(expectedVals[0]); + assertThat(vec.get(1)).isEqualTo(expectedVals[1]); } + @UseDataProvider("dataProvider") @Test - public void should_build_vector_from_elements() { - - validate_built_vector(CqlVector.newInstance(VECTOR_ARGS)); + public void should_build_vector_from_elements(Object[] vals) { + validate_built_vector(CqlVector.newInstance(vals), vals); } @Test - public void should_build_vector_from_list() { - - validate_built_vector(CqlVector.newInstance(Lists.newArrayList(VECTOR_ARGS))); - } - - @Test - public void should_build_vector_from_tostring_output() { - - CqlVector vector1 = CqlVector.newInstance(VECTOR_ARGS); - CqlVector vector2 = CqlVector.from(vector1.toString(), TypeCodecs.FLOAT); - assertThat(vector2).isEqualTo(vector1); + @UseDataProvider("dataProvider") + public void should_build_vector_from_list(Object[] vals) { + validate_built_vector(CqlVector.newInstance(Lists.newArrayList(vals)), vals); } @Test public void should_throw_from_null_string() { - assertThatThrownBy( () -> { CqlVector.from(null, TypeCodecs.FLOAT); @@ -116,101 +120,97 @@ public void should_throw_when_building_with_nulls() { @Test public void should_build_empty_vector() { - CqlVector vector = CqlVector.newInstance(); assertThat(vector.isEmpty()).isTrue(); assertThat(vector.size()).isEqualTo(0); } @Test - public void should_behave_mostly_like_a_list() { - - CqlVector vector = CqlVector.newInstance(VECTOR_ARGS); - assertThat(vector.get(0)).isEqualTo(VECTOR_ARGS[0]); - Float newVal = VECTOR_ARGS[0] * 2; - vector.set(0, newVal); - assertThat(vector.get(0)).isEqualTo(newVal); + @UseDataProvider("dataProvider") + public void should_behave_mostly_like_a_list(T[] vals) { + T[] theArray = Arrays.copyOf(vals, vals.length); + CqlVector vector = CqlVector.newInstance(theArray); + assertThat(vector.get(0)).isEqualTo(theArray[0]); + vector.set(0, theArray[1]); + assertThat(vector.get(0)).isEqualTo(theArray[1]); assertThat(vector.isEmpty()).isFalse(); assertThat(vector.size()).isEqualTo(2); - assertThat(Iterators.toArray(vector.iterator(), Float.class)).isEqualTo(VECTOR_ARGS); + Iterator iterator = vector.iterator(); + assertThat(iterator.next()).isEqualTo(theArray[1]); + assertThat(iterator.next()).isEqualTo(theArray[1]); } @Test - public void should_play_nicely_with_streams() { - - CqlVector vector = CqlVector.newInstance(VECTOR_ARGS); - List results = + @UseDataProvider("dataProvider") + public void should_play_nicely_with_streams(T[] vals) { + CqlVector vector = CqlVector.newInstance(vals); + List results = vector.stream() - .map((f) -> f * 2) - .collect(Collectors.toCollection(() -> new ArrayList())); + .map(Object::toString) + .collect(Collectors.toCollection(() -> new ArrayList())); for (int i = 0; i < vector.size(); ++i) { - assertThat(results.get(i)).isEqualTo(vector.get(i) * 2); + assertThat(results.get(i)).isEqualTo(vector.get(i).toString()); } } @Test - public void should_reflect_changes_to_mutable_list() { - - List theList = Lists.newArrayList(1.1f, 2.2f, 3.3f); - CqlVector vector = CqlVector.newInstance(theList); - assertThat(vector.size()).isEqualTo(3); - assertThat(vector.get(2)).isEqualTo(3.3f); - - float newVal1 = 4.4f; - theList.set(2, newVal1); - assertThat(vector.size()).isEqualTo(3); - assertThat(vector.get(2)).isEqualTo(newVal1); + @UseDataProvider("dataProvider") + public void should_reflect_changes_to_mutable_list(T[] vals) { + List theList = Lists.newArrayList(vals); + CqlVector vector = CqlVector.newInstance(theList); + assertThat(vector.size()).isEqualTo(2); + assertThat(vector.get(1)).isEqualTo(vals[1]); - float newVal2 = 5.5f; - theList.add(newVal2); - assertThat(vector.size()).isEqualTo(4); - assertThat(vector.get(3)).isEqualTo(newVal2); + T newVal = vals[0]; + theList.set(1, newVal); + assertThat(vector.size()).isEqualTo(2); + assertThat(vector.get(1)).isEqualTo(newVal); } @Test - public void should_reflect_changes_to_array() { - - Float[] theArray = new Float[] {1.1f, 2.2f, 3.3f}; - CqlVector vector = CqlVector.newInstance(theArray); - assertThat(vector.size()).isEqualTo(3); - assertThat(vector.get(2)).isEqualTo(3.3f); + @UseDataProvider("dataProvider") + public void should_reflect_changes_to_array(T[] vals) { + T[] theArray = Arrays.copyOf(vals, vals.length); + CqlVector vector = CqlVector.newInstance(theArray); + assertThat(vector.size()).isEqualTo(2); + assertThat(vector.get(1)).isEqualTo(theArray[1]); - float newVal1 = 4.4f; - theArray[2] = newVal1; - assertThat(vector.size()).isEqualTo(3); - assertThat(vector.get(2)).isEqualTo(newVal1); + T newVal = theArray[0]; + theArray[1] = newVal; + assertThat(vector.size()).isEqualTo(2); + assertThat(vector.get(1)).isEqualTo(newVal); } @Test - public void should_correctly_compare_vectors() { - - Float[] args = VECTOR_ARGS.clone(); - CqlVector vector1 = CqlVector.newInstance(args); - CqlVector vector2 = CqlVector.newInstance(args); - CqlVector vector3 = CqlVector.newInstance(Lists.newArrayList(args)); + @UseDataProvider("dataProvider") + public void should_correctly_compare_vectors(T[] vals) { + CqlVector vector1 = CqlVector.newInstance(vals); + CqlVector vector2 = CqlVector.newInstance(vals); + CqlVector vector3 = CqlVector.newInstance(Lists.newArrayList(vals)); assertThat(vector1).isNotSameAs(vector2); assertThat(vector1).isEqualTo(vector2); assertThat(vector1).isNotSameAs(vector3); assertThat(vector1).isEqualTo(vector3); - Float[] differentArgs = args.clone(); - float newVal = differentArgs[0] * 2; + T[] differentArgs = Arrays.copyOf(vals, vals.length); + T newVal = differentArgs[1]; differentArgs[0] = newVal; - CqlVector vector4 = CqlVector.newInstance(differentArgs); + CqlVector vector4 = CqlVector.newInstance(differentArgs); assertThat(vector1).isNotSameAs(vector4); assertThat(vector1).isNotEqualTo(vector4); - Float[] biggerArgs = Arrays.copyOf(args, args.length + 1); + T[] biggerArgs = Arrays.copyOf(vals, vals.length + 1); biggerArgs[biggerArgs.length - 1] = newVal; - CqlVector vector5 = CqlVector.newInstance(biggerArgs); + CqlVector vector5 = CqlVector.newInstance(biggerArgs); assertThat(vector1).isNotSameAs(vector5); assertThat(vector1).isNotEqualTo(vector5); } @Test - public void should_serialize_and_deserialize() throws Exception { - CqlVector initial = CqlVector.newInstance(VECTOR_ARGS); - CqlVector deserialized = SerializationHelper.serializeAndDeserialize(initial); + @UseDataProvider("dataProvider") + public void should_serialize_and_deserialize(T[] vals) throws Exception { + CqlVector initial = CqlVector.newInstance(vals); + CqlVector deserialized = SerializationHelper.serializeAndDeserialize(initial); assertThat(deserialized).isEqualTo(initial); } @@ -222,21 +222,22 @@ public void should_serialize_and_deserialize_empty_vector() throws Exception { } @Test - public void should_serialize_and_deserialize_unserializable_list() throws Exception { - CqlVector initial = + @UseDataProvider("dataProvider") + public void should_serialize_and_deserialize_unserializable_list(T[] vals) throws Exception { + CqlVector initial = CqlVector.newInstance( - new AbstractList() { + new AbstractList() { @Override - public Float get(int index) { - return VECTOR_ARGS[index]; + public T get(int index) { + return vals[index]; } @Override public int size() { - return VECTOR_ARGS.length; + return vals.length; } }); - CqlVector deserialized = SerializationHelper.serializeAndDeserialize(initial); + CqlVector deserialized = SerializationHelper.serializeAndDeserialize(initial); assertThat(deserialized).isEqualTo(initial); } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodecTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodecTest.java index 969d35cbbbe..17c78514127 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodecTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/VectorCodecTest.java @@ -20,122 +20,255 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.datastax.oss.driver.api.core.ProtocolVersion; import com.datastax.oss.driver.api.core.data.CqlVector; +import com.datastax.oss.driver.api.core.type.DataType; import com.datastax.oss.driver.api.core.type.DataTypes; -import com.datastax.oss.driver.api.core.type.VectorType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; -import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.datastax.oss.driver.internal.core.type.DefaultVectorType; -import java.util.Arrays; +import com.datastax.oss.protocol.internal.util.Bytes; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.nio.ByteBuffer; +import java.time.LocalTime; +import java.util.HashMap; +import org.apache.commons.lang3.ArrayUtils; import org.junit.Test; +import org.junit.runner.RunWith; -public class VectorCodecTest extends CodecTestBase> { +@RunWith(DataProviderRunner.class) +public class VectorCodecTest { - private static final Float[] VECTOR_ARGS = {1.0f, 2.5f}; - - private static final CqlVector VECTOR = CqlVector.newInstance(VECTOR_ARGS); - - private static final String VECTOR_HEX_STRING = "0x" + "3f800000" + "40200000"; - - private static final String FORMATTED_VECTOR = "[1.0, 2.5]"; - - public VectorCodecTest() { - VectorType vectorType = DataTypes.vectorOf(DataTypes.FLOAT, 2); - this.codec = TypeCodecs.vectorOf(vectorType, TypeCodecs.FLOAT); + @DataProvider + public static Object[] dataProvider() { + HashMap map1 = new HashMap<>(); + map1.put(1, "a"); + HashMap map2 = new HashMap<>(); + map2.put(2, "b"); + return new TestDataContainer[] { + new TestDataContainer( + DataTypes.FLOAT, + new Float[] {1.0f, 2.5f}, + "[1.0, 2.5]", + Bytes.fromHexString("0x3f80000040200000")), + new TestDataContainer( + DataTypes.ASCII, + new String[] {"ab", "cde"}, + "['ab', 'cde']", + Bytes.fromHexString("0x02616203636465")), + new TestDataContainer( + DataTypes.BIGINT, + new Long[] {1L, 2L}, + "[1, 2]", + Bytes.fromHexString("0x00000000000000010000000000000002")), + new TestDataContainer( + DataTypes.BLOB, + new ByteBuffer[] {Bytes.fromHexString("0xCAFE"), Bytes.fromHexString("0xABCD")}, + "[0xcafe, 0xabcd]", + Bytes.fromHexString("0x02cafe02abcd")), + new TestDataContainer( + DataTypes.BOOLEAN, + new Boolean[] {true, false}, + "[true, false]", + Bytes.fromHexString("0x0100")), + new TestDataContainer( + DataTypes.TIME, + new LocalTime[] {LocalTime.ofNanoOfDay(1), LocalTime.ofNanoOfDay(2)}, + "['00:00:00.000000001', '00:00:00.000000002']", + Bytes.fromHexString("0x080000000000000001080000000000000002")), + new TestDataContainer( + DataTypes.mapOf(DataTypes.INT, DataTypes.ASCII), + new HashMap[] {map1, map2}, + "[{1:'a'}, {2:'b'}]", + Bytes.fromHexString( + "0x110000000100000004000000010000000161110000000100000004000000020000000162")), + new TestDataContainer( + DataTypes.vectorOf(DataTypes.INT, 1), + new CqlVector[] {CqlVector.newInstance(1), CqlVector.newInstance(2)}, + "[[1], [2]]", + Bytes.fromHexString("0x0000000100000002")), + new TestDataContainer( + DataTypes.vectorOf(DataTypes.TEXT, 1), + new CqlVector[] {CqlVector.newInstance("ab"), CqlVector.newInstance("cdef")}, + "[['ab'], ['cdef']]", + Bytes.fromHexString("0x03026162050463646566")), + new TestDataContainer( + DataTypes.vectorOf(DataTypes.vectorOf(DataTypes.FLOAT, 2), 1), + new CqlVector[] { + CqlVector.newInstance(CqlVector.newInstance(1.0f, 2.5f)), + CqlVector.newInstance(CqlVector.newInstance(3.0f, 4.5f)) + }, + "[[[1.0, 2.5]], [[3.0, 4.5]]]", + Bytes.fromHexString("0x3f800000402000004040000040900000")) + }; } + @UseDataProvider("dataProvider") @Test - public void should_encode() { - assertThat(encode(VECTOR)).isEqualTo(VECTOR_HEX_STRING); - assertThat(encode(null)).isNull(); + public void should_encode(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + CqlVector vector = CqlVector.newInstance(testData.getValues()); + assertThat(codec.encode(vector, ProtocolVersion.DEFAULT)).isEqualTo(testData.getBytes()); } - /** Too few eleements will cause an exception, extra elements will be silently ignored */ @Test - public void should_throw_on_encode_with_too_few_elements() { - assertThatThrownBy(() -> encode(VECTOR.subVector(0, 1))) + @UseDataProvider("dataProvider") + public void should_throw_on_encode_with_too_few_elements(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThatThrownBy( + () -> + codec.encode( + CqlVector.newInstance(testData.getValues()[0]), ProtocolVersion.DEFAULT)) .isInstanceOf(IllegalArgumentException.class); } @Test - public void should_throw_on_encode_with_empty_list() { - assertThatThrownBy(() -> encode(CqlVector.newInstance())) + @UseDataProvider("dataProvider") + public void should_throw_on_encode_with_too_many_elements(TestDataContainer testData) { + Object[] doubled = ArrayUtils.addAll(testData.getValues(), testData.getValues()); + TypeCodec> codec = getCodec(testData.getDataType()); + assertThatThrownBy(() -> codec.encode(CqlVector.newInstance(doubled), ProtocolVersion.DEFAULT)) .isInstanceOf(IllegalArgumentException.class); } @Test - public void should_encode_with_too_many_elements() { - Float[] doubledVectorContents = Arrays.copyOf(VECTOR_ARGS, VECTOR_ARGS.length * 2); - System.arraycopy(VECTOR_ARGS, 0, doubledVectorContents, VECTOR_ARGS.length, VECTOR_ARGS.length); - assertThat(encode(CqlVector.newInstance(doubledVectorContents))).isEqualTo(VECTOR_HEX_STRING); + @UseDataProvider("dataProvider") + public void should_decode(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.decode(testData.getBytes(), ProtocolVersion.DEFAULT)) + .isEqualTo(CqlVector.newInstance(testData.getValues())); } @Test - public void should_decode() { - assertThat(decode(VECTOR_HEX_STRING)).isEqualTo(VECTOR); - assertThat(decode("0x")).isNull(); - assertThat(decode(null)).isNull(); + @UseDataProvider("dataProvider") + public void should_throw_on_decode_if_too_few_bytes(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + int lastIndex = testData.getBytes().remaining() - 1; + assertThatThrownBy( + () -> + codec.decode( + (ByteBuffer) testData.getBytes().duplicate().limit(lastIndex), + ProtocolVersion.DEFAULT)) + .isInstanceOf(IllegalArgumentException.class); } @Test - public void should_throw_on_decode_if_too_few_bytes() { - // Dropping 4 bytes would knock off exactly 1 float, anything less than that would be something - // we couldn't parse a float out of - for (int i = 1; i <= 3; ++i) { - // 2 chars of hex encoded string = 1 byte - int lastIndex = VECTOR_HEX_STRING.length() - (2 * i); - assertThatThrownBy(() -> decode(VECTOR_HEX_STRING.substring(0, lastIndex))) - .isInstanceOf(IllegalArgumentException.class); - } + @UseDataProvider("dataProvider") + public void should_throw_on_decode_if_too_many_bytes(TestDataContainer testData) { + ByteBuffer doubled = ByteBuffer.allocate(testData.getBytes().remaining() * 2); + doubled.put(testData.getBytes().duplicate()).put(testData.getBytes().duplicate()).flip(); + TypeCodec> codec = getCodec(testData.getDataType()); + assertThatThrownBy(() -> codec.decode(doubled, ProtocolVersion.DEFAULT)) + .isInstanceOf(IllegalArgumentException.class); } @Test - public void should_format() { - assertThat(format(VECTOR)).isEqualTo(FORMATTED_VECTOR); - assertThat(format(null)).isEqualTo("NULL"); + @UseDataProvider("dataProvider") + public void should_format(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + CqlVector vector = CqlVector.newInstance(testData.getValues()); + assertThat(codec.format(vector)).isEqualTo(testData.getFormatted()); } @Test - public void should_parse() { - assertThat(parse(FORMATTED_VECTOR)).isEqualTo(VECTOR); - assertThat(parse("NULL")).isNull(); - assertThat(parse("null")).isNull(); - assertThat(parse("")).isNull(); - assertThat(parse(null)).isNull(); + @UseDataProvider("dataProvider") + public void should_parse(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.parse(testData.getFormatted())) + .isEqualTo(CqlVector.newInstance(testData.getValues())); } @Test - public void should_accept_data_type() { - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, 2))).isTrue(); - assertThat(codec.accepts(DataTypes.INT)).isFalse(); + @UseDataProvider("dataProvider") + public void should_accept_data_type(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.accepts(new DefaultVectorType(testData.getDataType(), 2))).isTrue(); + assertThat(codec.accepts(new DefaultVectorType(DataTypes.custom("non-existent"), 2))).isFalse(); } @Test - public void should_accept_vector_type_correct_dimension_only() { - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, 0))).isFalse(); - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, 1))).isFalse(); - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, 2))).isTrue(); - for (int i = 3; i < 1000; ++i) { - assertThat(codec.accepts(new DefaultVectorType(DataTypes.FLOAT, i))).isFalse(); - } + @UseDataProvider("dataProvider") + public void should_accept_vector_type_correct_dimension_only(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.accepts(new DefaultVectorType(testData.getDataType(), 0))).isFalse(); + assertThat(codec.accepts(new DefaultVectorType(testData.getDataType(), 1))).isFalse(); + assertThat(codec.accepts(new DefaultVectorType(testData.getDataType(), 3))).isFalse(); } @Test - public void should_accept_generic_type() { - assertThat(codec.accepts(GenericType.vectorOf(GenericType.FLOAT))).isTrue(); - assertThat(codec.accepts(GenericType.vectorOf(GenericType.INTEGER))).isFalse(); - assertThat(codec.accepts(GenericType.of(Integer.class))).isFalse(); + @UseDataProvider("dataProvider") + public void should_accept_generic_type(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + assertThat(codec.accepts(codec.getJavaType())).isTrue(); } @Test - public void should_accept_raw_type() { + @UseDataProvider("dataProvider") + public void should_accept_raw_type(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); assertThat(codec.accepts(CqlVector.class)).isTrue(); assertThat(codec.accepts(Integer.class)).isFalse(); } @Test - public void should_accept_object() { - assertThat(codec.accepts(VECTOR)).isTrue(); + @UseDataProvider("dataProvider") + public void should_accept_object(TestDataContainer testData) { + TypeCodec> codec = getCodec(testData.getDataType()); + CqlVector vector = CqlVector.newInstance(testData.getValues()); + assertThat(codec.accepts(vector)).isTrue(); assertThat(codec.accepts(Integer.MIN_VALUE)).isFalse(); } + + @Test + public void should_handle_null_and_empty() { + TypeCodec> codec = getCodec(DataTypes.FLOAT); + assertThat(codec.encode(null, ProtocolVersion.DEFAULT)).isNull(); + assertThat(codec.decode(Bytes.fromHexString("0x"), ProtocolVersion.DEFAULT)).isNull(); + assertThat(codec.format(null)).isEqualTo("NULL"); + assertThat(codec.parse("NULL")).isNull(); + assertThat(codec.parse("null")).isNull(); + assertThat(codec.parse("")).isNull(); + assertThat(codec.parse(null)).isNull(); + assertThatThrownBy(() -> codec.encode(CqlVector.newInstance(), ProtocolVersion.DEFAULT)) + .isInstanceOf(IllegalArgumentException.class); + } + + private static TypeCodec> getCodec(DataType dataType) { + return TypeCodecs.vectorOf( + DataTypes.vectorOf(dataType, 2), CodecRegistry.DEFAULT.codecFor(dataType)); + } + + private static class TestDataContainer { + private final DataType dataType; + private final Object[] values; + private final String formatted; + private final ByteBuffer bytes; + + public TestDataContainer( + DataType dataType, Object[] values, String formatted, ByteBuffer bytes) { + this.dataType = dataType; + this.values = values; + this.formatted = formatted; + this.bytes = bytes; + } + + public DataType getDataType() { + return dataType; + } + + public Object[] getValues() { + return values; + } + + public String getFormatted() { + return formatted; + } + + public ByteBuffer getBytes() { + return bytes; + } + } } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistryTestDataProviders.java b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistryTestDataProviders.java index 1f3f6bbff97..4c0298bafad 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistryTestDataProviders.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/type/codec/registry/CachingCodecRegistryTestDataProviders.java @@ -337,6 +337,26 @@ public static Object[][] collectionsWithCqlAndJavaTypes() GenericType.vectorOf(BigInteger.class), CqlVector.newInstance(BigInteger.ONE) }, + // vector with arbitrary types + { + DataTypes.vectorOf(DataTypes.TEXT, 2), + GenericType.vectorOf(String.class), + GenericType.vectorOf(String.class), + CqlVector.newInstance("abc", "de") + }, + { + DataTypes.vectorOf(DataTypes.TIME, 2), + GenericType.vectorOf(LocalTime.class), + GenericType.vectorOf(LocalTime.class), + CqlVector.newInstance(LocalTime.MIDNIGHT, LocalTime.NOON) + }, + { + DataTypes.vectorOf(DataTypes.vectorOf(DataTypes.TINYINT, 2), 2), + GenericType.vectorOf(GenericType.vectorOf(Byte.class)), + GenericType.vectorOf(GenericType.vectorOf(Byte.class)), + CqlVector.newInstance( + CqlVector.newInstance((byte) 1, (byte) 2), CqlVector.newInstance((byte) 3, (byte) 4)) + }, }; } diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/type/util/VIntCodingTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/type/util/VIntCodingTest.java new file mode 100644 index 00000000000..b85d6d66844 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/type/util/VIntCodingTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.type.util; + +import static org.junit.Assert.assertEquals; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.nio.ByteBuffer; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(DataProviderRunner.class) +public class VIntCodingTest { + @DataProvider + public static Object[] roundTripTestValues() { + return new Integer[] { + Integer.MAX_VALUE + 1, + Integer.MAX_VALUE, + Integer.MAX_VALUE - 1, + Integer.MIN_VALUE, + Integer.MIN_VALUE + 1, + Integer.MIN_VALUE - 1, + 0, + -1, + 1 + }; + }; + + private static final long[] LONGS = + new long[] { + 53L, + 10201L, + 1097151L, + 168435455L, + 33251130335L, + 3281283447775L, + 417672546086779L, + 52057592037927932L, + 72057594037927937L + }; + + @Test + public void should_compute_unsigned_vint_size() { + for (int i = 0; i < LONGS.length; i++) { + long val = LONGS[i]; + assertEquals(i + 1, VIntCoding.computeUnsignedVIntSize(val)); + } + } + + @Test + @UseDataProvider("roundTripTestValues") + public void should_write_and_read_unsigned_vint_32(int value) { + ByteBuffer bb = ByteBuffer.allocate(9); + + VIntCoding.writeUnsignedVInt32(value, bb); + bb.flip(); + assertEquals(value, VIntCoding.getUnsignedVInt32(bb, 0)); + } + + @Test + @UseDataProvider("roundTripTestValues") + public void should_write_and_read_unsigned_vint(int value) { + ByteBuffer bb = ByteBuffer.allocate(9); + + VIntCoding.writeUnsignedVInt(value, bb); + bb.flip(); + assertEquals(value, VIntCoding.getUnsignedVInt(bb, 0)); + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/data/DataTypeIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/data/DataTypeIT.java index a33c8704876..e3d891454de 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/data/DataTypeIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/data/DataTypeIT.java @@ -32,6 +32,7 @@ import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.core.data.CqlDuration; +import com.datastax.oss.driver.api.core.data.CqlVector; import com.datastax.oss.driver.api.core.data.SettableByIndex; import com.datastax.oss.driver.api.core.data.SettableByName; import com.datastax.oss.driver.api.core.data.TupleValue; @@ -183,6 +184,7 @@ public static Object[][] typeSamples() { // 5) include map // 6) include tuple // 7) include udt + // 8) include vector return Arrays.stream(primitiveSamples) .flatMap( o -> { @@ -263,6 +265,30 @@ public static Object[][] typeSamples() { UdtValue udtValue2 = udt.newValue(1, o[1]); samples.add(new Object[] {udt, udtValue2}); + if (CCM_RULE.getCassandraVersion().compareTo(Version.parse("5.0")) >= 0) { + // vector of type + CqlVector vector = CqlVector.newInstance(o[1]); + samples.add(new Object[] {DataTypes.vectorOf(dataType, 1), vector}); + } + + return samples.stream(); + }) + .toArray(Object[][]::new); + } + + @DataProvider + public static Object[][] addVectors() { + Object[][] previousSamples = typeSamples(); + if (CCM_RULE.getCassandraVersion().compareTo(Version.parse("5.0")) < 0) return previousSamples; + return Arrays.stream(previousSamples) + .flatMap( + o -> { + List samples = new ArrayList<>(); + samples.add(o); + if (o[1] == null) return samples.stream(); + DataType dataType = (DataType) o[0]; + CqlVector vector = CqlVector.newInstance(o[1]); + samples.add(new Object[] {DataTypes.vectorOf(dataType, 1), vector}); return samples.stream(); }) .toArray(Object[][]::new); @@ -278,7 +304,7 @@ public static void createTable() { List columnData = new ArrayList<>(); - for (Object[] sample : typeSamples()) { + for (Object[] sample : addVectors()) { DataType dataType = (DataType) sample[0]; if (!typeToColumnName.containsKey(dataType)) { @@ -308,7 +334,7 @@ private static int nextKey() { return keyCounter.incrementAndGet(); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_simple_statement_using_format( DataType dataType, K value, K expectedPrimitiveValue) { @@ -335,7 +361,7 @@ public void should_insert_non_primary_key_column_simple_statement_using_form readValue(select, dataType, value, expectedPrimitiveValue); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_simple_statement_positional_value( DataType dataType, K value, K expectedPrimitiveValue) { @@ -358,7 +384,7 @@ public void should_insert_non_primary_key_column_simple_statement_positional readValue(select, dataType, value, expectedPrimitiveValue); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_simple_statement_named_value( DataType dataType, K value, K expectedPrimitiveValue) { @@ -382,7 +408,7 @@ public void should_insert_non_primary_key_column_simple_statement_named_valu readValue(select, dataType, value, expectedPrimitiveValue); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_bound_statement_positional_value( DataType dataType, K value, K expectedPrimitiveValue) { @@ -411,7 +437,7 @@ public void should_insert_non_primary_key_column_bound_statement_positional_ readValue(boundSelect, dataType, value, expectedPrimitiveValue); } - @UseDataProvider("typeSamples") + @UseDataProvider("addVectors") @Test public void should_insert_non_primary_key_column_bound_statement_named_value( DataType dataType, K value, K expectedPrimitiveValue) { From 04d34a8989b89c1addaab7f248b1fa9aa535da5e Mon Sep 17 00:00:00 2001 From: Alex Sasnouskikh Date: Thu, 30 Jan 2025 22:36:29 +0000 Subject: [PATCH 37/48] JAVA-3168 Copy node info for contact points on initial node refresh only from first match by endpoint patch by Alex Sasnouskikh; reviewed by Andy Tolbert and Alexandre Dura for JAVA-3168 --- .../core/metadata/InitialNodeListRefresh.java | 36 +++++++++++-------- .../metadata/InitialNodeListRefreshTest.java | 34 ++++++++++++++++-- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefresh.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefresh.java index c21d5d8171e..517bfca27fa 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefresh.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefresh.java @@ -18,14 +18,16 @@ package com.datastax.oss.driver.internal.core.metadata; import com.datastax.oss.driver.api.core.metadata.EndPoint; -import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.metadata.token.TokenFactory; import com.datastax.oss.driver.internal.core.metadata.token.TokenFactoryRegistry; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -63,22 +65,31 @@ public Result compute( TokenFactory tokenFactory = null; Map newNodes = new HashMap<>(); + // Contact point nodes don't have host ID as well as other info yet, so we fill them with node + // info found on first match by endpoint + Set matchedContactPoints = new HashSet<>(); + List addedNodes = new ArrayList<>(); for (NodeInfo nodeInfo : nodeInfos) { UUID hostId = nodeInfo.getHostId(); if (newNodes.containsKey(hostId)) { LOG.warn( "[{}] Found duplicate entries with host_id {} in system.peers, " - + "keeping only the first one", + + "keeping only the first one {}", logPrefix, - hostId); + hostId, + newNodes.get(hostId)); } else { EndPoint endPoint = nodeInfo.getEndPoint(); - DefaultNode node = findIn(contactPoints, endPoint); - if (node == null) { + DefaultNode contactPointNode = findContactPointNode(endPoint); + DefaultNode node; + if (contactPointNode == null || matchedContactPoints.contains(endPoint)) { node = new DefaultNode(endPoint, context); + addedNodes.add(node); LOG.debug("[{}] Adding new node {}", logPrefix, node); } else { + matchedContactPoints.add(contactPointNode.getEndPoint()); + node = contactPointNode; LOG.debug("[{}] Copying contact point {}", logPrefix, node); } if (tokenMapEnabled && tokenFactory == null && nodeInfo.getPartitioner() != null) { @@ -90,14 +101,11 @@ public Result compute( } ImmutableList.Builder eventsBuilder = ImmutableList.builder(); - - for (DefaultNode newNode : newNodes.values()) { - if (findIn(contactPoints, newNode.getEndPoint()) == null) { - eventsBuilder.add(NodeStateEvent.added(newNode)); - } + for (DefaultNode addedNode : addedNodes) { + eventsBuilder.add(NodeStateEvent.added(addedNode)); } for (DefaultNode contactPoint : contactPoints) { - if (findIn(newNodes.values(), contactPoint.getEndPoint()) == null) { + if (!matchedContactPoints.contains(contactPoint.getEndPoint())) { eventsBuilder.add(NodeStateEvent.removed(contactPoint)); } } @@ -108,10 +116,10 @@ public Result compute( eventsBuilder.build()); } - private DefaultNode findIn(Iterable nodes, EndPoint endPoint) { - for (Node node : nodes) { + private DefaultNode findContactPointNode(EndPoint endPoint) { + for (DefaultNode node : contactPoints) { if (node.getEndPoint().equals(endPoint)) { - return (DefaultNode) node; + return node; } } return null; diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefreshTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefreshTest.java index 095662257f6..3787bf8fe10 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefreshTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/InitialNodeListRefreshTest.java @@ -48,6 +48,8 @@ public class InitialNodeListRefreshTest { private UUID hostId1; private UUID hostId2; private UUID hostId3; + private UUID hostId4; + private UUID hostId5; @Before public void setup() { @@ -61,10 +63,12 @@ public void setup() { hostId1 = UUID.randomUUID(); hostId2 = UUID.randomUUID(); hostId3 = UUID.randomUUID(); + hostId4 = UUID.randomUUID(); + hostId5 = UUID.randomUUID(); } @Test - public void should_copy_contact_points() { + public void should_copy_contact_points_on_first_endpoint_match_only() { // Given Iterable newInfos = ImmutableList.of( @@ -76,6 +80,17 @@ public void should_copy_contact_points() { DefaultNodeInfo.builder() .withEndPoint(contactPoint2.getEndPoint()) .withHostId(hostId2) + .build(), + DefaultNodeInfo.builder().withEndPoint(endPoint3).withHostId(hostId3).build(), + DefaultNodeInfo.builder() + // address translator can translate node addresses to the same endpoints + .withEndPoint(contactPoint2.getEndPoint()) + .withHostId(hostId4) + .build(), + DefaultNodeInfo.builder() + // address translator can translate node addresses to the same endpoints + .withEndPoint(endPoint3) + .withHostId(hostId5) .build()); InitialNodeListRefresh refresh = new InitialNodeListRefresh(newInfos, ImmutableSet.of(contactPoint1, contactPoint2)); @@ -86,11 +101,26 @@ public void should_copy_contact_points() { // Then // contact points have been copied to the metadata, and completed with missing information Map newNodes = result.newMetadata.getNodes(); - assertThat(newNodes).containsOnlyKeys(hostId1, hostId2); + assertThat(newNodes).containsOnlyKeys(hostId1, hostId2, hostId3, hostId4, hostId5); assertThat(newNodes.get(hostId1)).isEqualTo(contactPoint1); assertThat(contactPoint1.getHostId()).isEqualTo(hostId1); assertThat(newNodes.get(hostId2)).isEqualTo(contactPoint2); assertThat(contactPoint2.getHostId()).isEqualTo(hostId2); + // And + // node has been added for the new endpoint + assertThat(newNodes.get(hostId3).getEndPoint()).isEqualTo(endPoint3); + assertThat(newNodes.get(hostId3).getHostId()).isEqualTo(hostId3); + // And + // nodes have been added for duplicated endpoints + assertThat(newNodes.get(hostId4).getEndPoint()).isEqualTo(contactPoint2.getEndPoint()); + assertThat(newNodes.get(hostId4).getHostId()).isEqualTo(hostId4); + assertThat(newNodes.get(hostId5).getEndPoint()).isEqualTo(endPoint3); + assertThat(newNodes.get(hostId5).getHostId()).isEqualTo(hostId5); + assertThat(result.events) + .containsExactlyInAnyOrder( + NodeStateEvent.added((DefaultNode) newNodes.get(hostId3)), + NodeStateEvent.added((DefaultNode) newNodes.get(hostId4)), + NodeStateEvent.added((DefaultNode) newNodes.get(hostId5))); } @Test From eac7b24d60d87e912ea70ba3ddee2755a6fa28cf Mon Sep 17 00:00:00 2001 From: Luc Boutier Date: Fri, 3 Nov 2023 10:13:13 +0100 Subject: [PATCH 38/48] JAVA-3055: Prevent PreparedStatement cache to be polluted if a request is cancelled. There was a critical issue when the external code cancels a request, indeed the cached CompletableFuture will then always throw a CancellationException. This may happens, for example, when used by reactive like Mono.zip or Mono.firstWithValue. patch by Luc Boutier; reviewed by Alexandre Dutra and Bret McGuire reference: https://github.com/apache/cassandra-java-driver/pull/1757 --- .../driver/internal/core/cql/CqlPrepareAsyncProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java index ffbc8ee046a..d777e35e50f 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java @@ -159,7 +159,9 @@ public CompletionStage process( }); } } - return result; + // Return a defensive copy. So if a client cancels its request, the cache won't be impacted + // nor a potential concurrent request. + return result.thenApply(x -> x); // copy() is available only since Java 9 } catch (ExecutionException e) { return CompletableFutures.failedFuture(e.getCause()); } From 64b3568dfa6dd58e8c16d28cafbe8dbf16f1e622 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Mon, 3 Feb 2025 17:56:54 -0600 Subject: [PATCH 39/48] Expose a decorator for CqlPrepareAsyncProcessor cache rather than the ability to specify an arbitrary cache from scratch. Also bringing tests from https://github.com/apache/cassandra-java-driver/pull/2003 forward with a few minor changes due to this implementation patch by Bret McGuire; reviewed by Bret McGuire and Andy Tolbert reference: #2008 --- .../core/cql/CqlPrepareAsyncProcessor.java | 11 +- .../core/cql/PreparedStatementCachingIT.java | 13 +- .../cql/PreparedStatementCancellationIT.java | 166 ++++++++++++++++++ 3 files changed, 178 insertions(+), 12 deletions(-) create mode 100644 integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCancellationIT.java diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java index d777e35e50f..918d75e9ecb 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java @@ -38,6 +38,7 @@ import com.datastax.oss.driver.shaded.guava.common.cache.CacheBuilder; import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; import com.datastax.oss.protocol.internal.ProtocolConstants; +import com.google.common.base.Functions; import edu.umd.cs.findbugs.annotations.NonNull; import io.netty.util.concurrent.EventExecutor; import java.util.Map; @@ -45,6 +46,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; +import java.util.function.Function; import net.jcip.annotations.ThreadSafe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,14 +64,15 @@ public CqlPrepareAsyncProcessor() { } public CqlPrepareAsyncProcessor(@NonNull Optional context) { - this(CacheBuilder.newBuilder().weakValues().build(), context); + this(context, Functions.identity()); } protected CqlPrepareAsyncProcessor( - Cache> cache, - Optional context) { + Optional context, + Function, CacheBuilder> decorator) { - this.cache = cache; + CacheBuilder baseCache = CacheBuilder.newBuilder().weakValues(); + this.cache = decorator.apply(baseCache).build(); context.ifPresent( (ctx) -> { LOG.info("Adding handler to invalidate cached prepared statements on type changes"); diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java index 05ac3bd0e92..617d489fb95 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCachingIT.java @@ -24,7 +24,6 @@ import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; import com.datastax.oss.driver.api.core.context.DriverContext; -import com.datastax.oss.driver.api.core.cql.PrepareRequest; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.metrics.DefaultSessionMetric; import com.datastax.oss.driver.api.core.session.ProgrammaticArguments; @@ -41,7 +40,6 @@ import com.datastax.oss.driver.internal.core.session.BuiltInRequestProcessors; import com.datastax.oss.driver.internal.core.session.RequestProcessor; import com.datastax.oss.driver.internal.core.session.RequestProcessorRegistry; -import com.datastax.oss.driver.shaded.guava.common.cache.CacheBuilder; import com.datastax.oss.driver.shaded.guava.common.cache.RemovalListener; import com.datastax.oss.driver.shaded.guava.common.util.concurrent.Uninterruptibles; import com.google.common.collect.ImmutableList; @@ -119,11 +117,12 @@ private static class TestCqlPrepareAsyncProcessor extends CqlPrepareAsyncProcess private static final Logger LOG = LoggerFactory.getLogger(PreparedStatementCachingIT.TestCqlPrepareAsyncProcessor.class); - private static RemovalListener> - buildCacheRemoveCallback(@NonNull Optional context) { + private static RemovalListener buildCacheRemoveCallback( + @NonNull Optional context) { return (evt) -> { try { - CompletableFuture future = evt.getValue(); + CompletableFuture future = + (CompletableFuture) evt.getValue(); ByteBuffer queryId = Uninterruptibles.getUninterruptibly(future).getId(); context.ifPresent( ctx -> ctx.getEventBus().fire(new PreparedStatementRemovalEvent(queryId))); @@ -136,9 +135,7 @@ private static class TestCqlPrepareAsyncProcessor extends CqlPrepareAsyncProcess public TestCqlPrepareAsyncProcessor(@NonNull Optional context) { // Default CqlPrepareAsyncProcessor uses weak values here as well. We avoid doing so // to prevent cache entries from unexpectedly disappearing mid-test. - super( - CacheBuilder.newBuilder().removalListener(buildCacheRemoveCallback(context)).build(), - context); + super(context, builder -> builder.removalListener(buildCacheRemoveCallback(context))); } } diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCancellationIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCancellationIT.java new file mode 100644 index 00000000000..d7e581e4606 --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/PreparedStatementCancellationIT.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.core.cql; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.fail; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.PrepareRequest; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.session.SessionRule; +import com.datastax.oss.driver.api.testinfra.session.SessionUtils; +import com.datastax.oss.driver.categories.IsolatedTests; +import com.datastax.oss.driver.internal.core.context.DefaultDriverContext; +import com.datastax.oss.driver.internal.core.cql.CqlPrepareAsyncProcessor; +import com.datastax.oss.driver.shaded.guava.common.base.Predicates; +import com.datastax.oss.driver.shaded.guava.common.cache.Cache; +import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; +import java.util.concurrent.CompletableFuture; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +@Category(IsolatedTests.class) +public class PreparedStatementCancellationIT { + + private CustomCcmRule ccmRule = CustomCcmRule.builder().build(); + + private SessionRule sessionRule = SessionRule.builder(ccmRule).build(); + + @Rule public TestRule chain = RuleChain.outerRule(ccmRule).around(sessionRule); + + @Before + public void setup() { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + session.execute("DROP TABLE IF EXISTS test_table_1"); + session.execute("CREATE TABLE test_table_1 (k int primary key, v int)"); + session.execute("INSERT INTO test_table_1 (k,v) VALUES (1, 100)"); + session.execute("INSERT INTO test_table_1 (k,v) VALUES (2, 200)"); + session.execute("INSERT INTO test_table_1 (k,v) VALUES (3, 300)"); + session.close(); + } + + @After + public void teardown() { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + session.execute("DROP TABLE test_table_1"); + session.close(); + } + + private CompletableFuture toCompletableFuture(CqlSession session, String cql) { + + return session.prepareAsync(cql).toCompletableFuture(); + } + + private CqlPrepareAsyncProcessor findProcessor(CqlSession session) { + + DefaultDriverContext context = (DefaultDriverContext) session.getContext(); + return (CqlPrepareAsyncProcessor) + Iterables.find( + context.getRequestProcessorRegistry().getProcessors(), + Predicates.instanceOf(CqlPrepareAsyncProcessor.class)); + } + + @Test + public void should_cache_valid_cql() throws Exception { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + CqlPrepareAsyncProcessor processor = findProcessor(session); + Cache> cache = processor.getCache(); + assertThat(cache.size()).isEqualTo(0); + + // Make multiple CompletableFuture requests for the specified CQL, then wait until + // the cached request finishes and confirm that all futures got the same values + String cql = "select v from test_table_1 where k = ?"; + CompletableFuture cf1 = toCompletableFuture(session, cql); + CompletableFuture cf2 = toCompletableFuture(session, cql); + assertThat(cache.size()).isEqualTo(1); + + CompletableFuture future = Iterables.get(cache.asMap().values(), 0); + PreparedStatement stmt = future.get(); + + assertThat(cf1.isDone()).isTrue(); + assertThat(cf2.isDone()).isTrue(); + + assertThat(cf1.join()).isEqualTo(stmt); + assertThat(cf2.join()).isEqualTo(stmt); + } + + // A holdover from work done on JAVA-3055. This probably isn't _desired_ behaviour but this test + // documents the fact that the current driver impl will behave in this way. We should probably + // consider changing this in a future release, although it's worthwhile fully considering the + // implications of such a change. + @Test + public void will_cache_invalid_cql() throws Exception { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + CqlPrepareAsyncProcessor processor = findProcessor(session); + Cache> cache = processor.getCache(); + assertThat(cache.size()).isEqualTo(0); + + // Verify that we get the CompletableFuture even if the CQL is invalid but that nothing is + // cached + String cql = "select v fromfrom test_table_1 where k = ?"; + CompletableFuture cf = toCompletableFuture(session, cql); + + // join() here should throw exceptions due to the invalid syntax... for purposes of this test we + // can ignore this + try { + cf.join(); + fail(); + } catch (Exception e) { + } + + assertThat(cache.size()).isEqualTo(1); + } + + @Test + public void should_not_affect_cache_if_returned_futures_are_cancelled() throws Exception { + + CqlSession session = SessionUtils.newSession(ccmRule, sessionRule.keyspace()); + CqlPrepareAsyncProcessor processor = findProcessor(session); + Cache> cache = processor.getCache(); + assertThat(cache.size()).isEqualTo(0); + + String cql = "select v from test_table_1 where k = ?"; + CompletableFuture cf = toCompletableFuture(session, cql); + + assertThat(cf.isCancelled()).isFalse(); + assertThat(cf.cancel(false)).isTrue(); + assertThat(cf.isCancelled()).isTrue(); + assertThat(cf.isCompletedExceptionally()).isTrue(); + + // Confirm that cancelling the CompletableFuture returned to the user does _not_ cancel the + // future used within the cache. CacheEntry very deliberately doesn't maintain a reference + // to it's contained CompletableFuture so we have to get at this by secondary effects. + assertThat(cache.size()).isEqualTo(1); + CompletableFuture future = Iterables.get(cache.asMap().values(), 0); + PreparedStatement rv = future.get(); + assertThat(rv).isNotNull(); + assertThat(rv.getQuery()).isEqualTo(cql); + assertThat(cache.size()).isEqualTo(1); + } +} From 94e73d9f4e95cd74b319495663862ad50c63f589 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Wed, 5 Feb 2025 15:33:46 -0600 Subject: [PATCH 40/48] ninja-fix Using shaded Guava classes for import in order to make OSGi class paths happy. Major hat tip to Dmitry Konstantinov for the find here! --- .../oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java index 918d75e9ecb..a3d11cff054 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlPrepareAsyncProcessor.java @@ -34,11 +34,11 @@ import com.datastax.oss.driver.internal.core.session.RequestProcessor; import com.datastax.oss.driver.internal.core.util.concurrent.CompletableFutures; import com.datastax.oss.driver.internal.core.util.concurrent.RunOrSchedule; +import com.datastax.oss.driver.shaded.guava.common.base.Functions; import com.datastax.oss.driver.shaded.guava.common.cache.Cache; import com.datastax.oss.driver.shaded.guava.common.cache.CacheBuilder; import com.datastax.oss.driver.shaded.guava.common.collect.Iterables; import com.datastax.oss.protocol.internal.ProtocolConstants; -import com.google.common.base.Functions; import edu.umd.cs.findbugs.annotations.NonNull; import io.netty.util.concurrent.EventExecutor; import java.util.Map; From 610b91b160948c96c2d61aa34021db3cdac73aa2 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Tue, 4 Feb 2025 15:22:14 -0600 Subject: [PATCH 41/48] Changelog updates for 4.19.0 --- changelog/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/changelog/README.md b/changelog/README.md index 83ebb44239f..08634bcb834 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -21,6 +21,30 @@ under the License. +### 4.19.0 + +- [bug] JAVA-3055: Prevent PreparedStatement cache to be polluted if a request is cancelled. +- [bug] JAVA-3168: Copy node info for contact points on initial node refresh only from first match by endpoint +- [improvement] JAVA-3143: Extend driver vector support to arbitrary subtypes and fix handling of variable length types (OSS C* 5.0) +- [improvement] CASSJAVA-53: Update Guava version used in cassandra-java-driver +- [improvement] JAVA-3118: Add support for vector data type in Schema Builder, QueryBuilder +- [bug] CASSJAVA-55: Remove setting "Host" header for metadata requests +- [bug] JAVA-3057: Allow decoding a UDT that has more fields than expected +- [improvement] CASSJAVA-52: Bring java-driver-shaded-guava into the repo as a submodule +- [bug] CASSJAVA-2: TableMetadata#describe produces invalid CQL when a type of a column is a vector +- [bug] JAVA-3051: Memory leak in DefaultLoadBalancingPolicy measurement of response times +- [improvement] CASSJAVA-14: Query builder support for NOT CQL syntax +- [bug] CASSJAVA-12: DefaultSslEngineFactory missing null check on close +- [improvement] CASSJAVA-46: Expose table extensions via schema builders +- [bug] PR 1938: Fix uncaught exception during graceful channel shutdown after exceeding max orphan ids +- [improvement] PR 1607: Annotate BatchStatement, Statement, SimpleStatement methods with CheckReturnValue +- [improvement] CASSJAVA-41: Reduce lock held duration in ConcurrencyLimitingRequestThrottler +- [bug] JAVA-3149: Async Query Cancellation Not Propagated To RequestThrottler +- [bug] JAVA-3167: CompletableFutures.allSuccessful() may return never completed future +- [bug] PR 1620: Don't return empty routing key when partition key is unbound +- [improvement] PR 1623: Limit calls to Conversions.resolveExecutionProfile +- [improvement] CASSJAVA-29: Update target Cassandra versions for integration tests, support new 5.0.x + ### 4.18.1 - [improvement] JAVA-3142: Ability to specify ordering of remote local dc's via new configuration for graceful automatic failovers From 46444eaabdbd23e9231123198536d070e99aca27 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Thu, 6 Feb 2025 15:51:51 -0600 Subject: [PATCH 42/48] [maven-release-plugin] prepare release 4.19.0 --- bom/pom.xml | 20 ++++++++++---------- core-shaded/pom.xml | 2 +- core/pom.xml | 2 +- distribution-source/pom.xml | 2 +- distribution-tests/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- guava-shaded/pom.xml | 2 +- integration-tests/pom.xml | 2 +- mapper-processor/pom.xml | 2 +- mapper-runtime/pom.xml | 2 +- metrics/micrometer/pom.xml | 2 +- metrics/microprofile/pom.xml | 2 +- osgi-tests/pom.xml | 2 +- pom.xml | 4 ++-- query-builder/pom.xml | 2 +- test-infra/pom.xml | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index 08f212f6157..bb39a484f71 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-bom pom @@ -33,47 +33,47 @@ org.apache.cassandra java-driver-core - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-core-shaded - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-mapper-processor - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-mapper-runtime - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-query-builder - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-guava-shaded - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-test-infra - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-metrics-micrometer - 4.18.2-SNAPSHOT + 4.19.0 org.apache.cassandra java-driver-metrics-microprofile - 4.18.2-SNAPSHOT + 4.19.0 com.datastax.oss diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 9a708beb2a7..82f45050098 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-core-shaded Apache Cassandra Java Driver - core with shaded deps diff --git a/core/pom.xml b/core/pom.xml index 2a48e8bf9ce..ea0efdddc08 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-core bundle diff --git a/distribution-source/pom.xml b/distribution-source/pom.xml index ee5b52958c3..d3275565827 100644 --- a/distribution-source/pom.xml +++ b/distribution-source/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-distribution-source pom diff --git a/distribution-tests/pom.xml b/distribution-tests/pom.xml index fafd8c4678b..0bb2ccef386 100644 --- a/distribution-tests/pom.xml +++ b/distribution-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-distribution-tests Apache Cassandra Java Driver - distribution tests diff --git a/distribution/pom.xml b/distribution/pom.xml index dfc406baf43..4edc44d2131 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-distribution diff --git a/examples/pom.xml b/examples/pom.xml index a76cc8d2bf1..ee90d5c2c65 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -23,7 +23,7 @@ java-driver-parent org.apache.cassandra - 4.18.2-SNAPSHOT + 4.19.0 java-driver-examples Apache Cassandra Java Driver - examples. diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index ca8f0161b04..a0627c376ff 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-guava-shaded Apache Cassandra Java Driver - guava shaded dep diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index d1b0a736bb0..1052f96ae35 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-integration-tests jar diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index 6588f17d5f7..0b201db2473 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-mapper-processor Apache Cassandra Java Driver - object mapper processor diff --git a/mapper-runtime/pom.xml b/mapper-runtime/pom.xml index 28483ee93ff..f846fc68dec 100644 --- a/mapper-runtime/pom.xml +++ b/mapper-runtime/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-mapper-runtime bundle diff --git a/metrics/micrometer/pom.xml b/metrics/micrometer/pom.xml index 8ab939cbb37..80ec51605ab 100644 --- a/metrics/micrometer/pom.xml +++ b/metrics/micrometer/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 ../../ java-driver-metrics-micrometer diff --git a/metrics/microprofile/pom.xml b/metrics/microprofile/pom.xml index 521a67f9075..955160dd621 100644 --- a/metrics/microprofile/pom.xml +++ b/metrics/microprofile/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 ../../ java-driver-metrics-microprofile diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index f0e66b656ca..7b761aa12a0 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-osgi-tests jar diff --git a/pom.xml b/pom.xml index c61e6485fd3..ecb800b7ebb 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 pom Apache Cassandra Java Driver https://github.com/datastax/java-driver @@ -1029,7 +1029,7 @@ limitations under the License.]]> scm:git:git@github.com:datastax/java-driver.git scm:git:git@github.com:datastax/java-driver.git https://github.com/datastax/java-driver - HEAD + 4.19.0 diff --git a/query-builder/pom.xml b/query-builder/pom.xml index 4e09a10e584..144906adecb 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-query-builder bundle diff --git a/test-infra/pom.xml b/test-infra/pom.xml index 262627e5536..0252683ca2b 100644 --- a/test-infra/pom.xml +++ b/test-infra/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.18.2-SNAPSHOT + 4.19.0 java-driver-test-infra bundle From 90612f6758eb0f0ba964daf054f397a47a90a736 Mon Sep 17 00:00:00 2001 From: absurdfarce Date: Thu, 6 Feb 2025 15:51:54 -0600 Subject: [PATCH 43/48] [maven-release-plugin] prepare for next development iteration --- bom/pom.xml | 20 ++++++++++---------- core-shaded/pom.xml | 2 +- core/pom.xml | 2 +- distribution-source/pom.xml | 2 +- distribution-tests/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- guava-shaded/pom.xml | 2 +- integration-tests/pom.xml | 2 +- mapper-processor/pom.xml | 2 +- mapper-runtime/pom.xml | 2 +- metrics/micrometer/pom.xml | 2 +- metrics/microprofile/pom.xml | 2 +- osgi-tests/pom.xml | 2 +- pom.xml | 4 ++-- query-builder/pom.xml | 2 +- test-infra/pom.xml | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bom/pom.xml b/bom/pom.xml index bb39a484f71..8e7bc467fe7 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-bom pom @@ -33,47 +33,47 @@ org.apache.cassandra java-driver-core - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-core-shaded - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-mapper-processor - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-mapper-runtime - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-query-builder - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-guava-shaded - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-test-infra - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-metrics-micrometer - 4.19.0 + 4.19.1-SNAPSHOT org.apache.cassandra java-driver-metrics-microprofile - 4.19.0 + 4.19.1-SNAPSHOT com.datastax.oss diff --git a/core-shaded/pom.xml b/core-shaded/pom.xml index 82f45050098..451db1dcd1b 100644 --- a/core-shaded/pom.xml +++ b/core-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-core-shaded Apache Cassandra Java Driver - core with shaded deps diff --git a/core/pom.xml b/core/pom.xml index ea0efdddc08..b8d7d5c2d3b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-core bundle diff --git a/distribution-source/pom.xml b/distribution-source/pom.xml index d3275565827..e930f4c0610 100644 --- a/distribution-source/pom.xml +++ b/distribution-source/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-distribution-source pom diff --git a/distribution-tests/pom.xml b/distribution-tests/pom.xml index 0bb2ccef386..1c762074673 100644 --- a/distribution-tests/pom.xml +++ b/distribution-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-distribution-tests Apache Cassandra Java Driver - distribution tests diff --git a/distribution/pom.xml b/distribution/pom.xml index 4edc44d2131..8f7740e148f 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-distribution diff --git a/examples/pom.xml b/examples/pom.xml index ee90d5c2c65..15f082e6864 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -23,7 +23,7 @@ java-driver-parent org.apache.cassandra - 4.19.0 + 4.19.1-SNAPSHOT java-driver-examples Apache Cassandra Java Driver - examples. diff --git a/guava-shaded/pom.xml b/guava-shaded/pom.xml index a0627c376ff..8053af94911 100644 --- a/guava-shaded/pom.xml +++ b/guava-shaded/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-guava-shaded Apache Cassandra Java Driver - guava shaded dep diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 1052f96ae35..b489076c257 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-integration-tests jar diff --git a/mapper-processor/pom.xml b/mapper-processor/pom.xml index 0b201db2473..519f1411ce9 100644 --- a/mapper-processor/pom.xml +++ b/mapper-processor/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-mapper-processor Apache Cassandra Java Driver - object mapper processor diff --git a/mapper-runtime/pom.xml b/mapper-runtime/pom.xml index f846fc68dec..3a767c2a352 100644 --- a/mapper-runtime/pom.xml +++ b/mapper-runtime/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-mapper-runtime bundle diff --git a/metrics/micrometer/pom.xml b/metrics/micrometer/pom.xml index 80ec51605ab..091bb5f3e93 100644 --- a/metrics/micrometer/pom.xml +++ b/metrics/micrometer/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT ../../ java-driver-metrics-micrometer diff --git a/metrics/microprofile/pom.xml b/metrics/microprofile/pom.xml index 955160dd621..5163b5366f4 100644 --- a/metrics/microprofile/pom.xml +++ b/metrics/microprofile/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT ../../ java-driver-metrics-microprofile diff --git a/osgi-tests/pom.xml b/osgi-tests/pom.xml index 7b761aa12a0..4b41f790145 100644 --- a/osgi-tests/pom.xml +++ b/osgi-tests/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-osgi-tests jar diff --git a/pom.xml b/pom.xml index ecb800b7ebb..3fd2d1347c2 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT pom Apache Cassandra Java Driver https://github.com/datastax/java-driver @@ -1029,7 +1029,7 @@ limitations under the License.]]> scm:git:git@github.com:datastax/java-driver.git scm:git:git@github.com:datastax/java-driver.git https://github.com/datastax/java-driver - 4.19.0 + HEAD diff --git a/query-builder/pom.xml b/query-builder/pom.xml index 144906adecb..0bc46f9bb91 100644 --- a/query-builder/pom.xml +++ b/query-builder/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-query-builder bundle diff --git a/test-infra/pom.xml b/test-infra/pom.xml index 0252683ca2b..b0808757ce4 100644 --- a/test-infra/pom.xml +++ b/test-infra/pom.xml @@ -23,7 +23,7 @@ org.apache.cassandra java-driver-parent - 4.19.0 + 4.19.1-SNAPSHOT java-driver-test-infra bundle From 33274ec7b59e6d71b7a322af38b78d068fd9fb8a Mon Sep 17 00:00:00 2001 From: Bouncheck <36934780+Bouncheck@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:44:45 +0100 Subject: [PATCH 44/48] Specify maven-clean-plugin version Sets the version to 3.4.1 in parent pom. Having it unspecified causes the following warning: ``` [WARNING] Some problems were encountered while building the effective model for com.scylladb:java-driver-guava-shaded:jar:4.19.0.0-SNAPSHOT [WARNING] 'build.plugins.plugin.version' for org.apache.maven.plugins:maven-clean-plugin is missing. @ line 97, column 15 [WARNING] [WARNING] It is highly recommended to fix these problems because they threaten the stability of your build. [WARNING] [WARNING] For this reason, future Maven versions might no longer support building such malformed projects. [WARNING] ``` --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index 6b88647b4bd..fcd7644a16e 100644 --- a/pom.xml +++ b/pom.xml @@ -475,6 +475,11 @@ maven-compiler-plugin 3.8.1 + + org.apache.maven.plugins + maven-clean-plugin + 3.4.1 + com.coveo fmt-maven-plugin From c1c1063a36ac663ffb65a12d6b0fb678991585c5 Mon Sep 17 00:00:00 2001 From: Bouncheck <36934780+Bouncheck@users.noreply.github.com> Date: Mon, 17 Mar 2025 12:09:24 +0100 Subject: [PATCH 45/48] Install guava-shaded before running core's compile in CI Without this module available "Build" and "Unit tests" job fail with "package does not exist" or "cannot find symbol" pointing to `[...].shaded.guava.[...]` packages. --- .github/workflows/tests@v1.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/tests@v1.yml b/.github/workflows/tests@v1.yml index 37e09ebf89a..721da1db0df 100644 --- a/.github/workflows/tests@v1.yml +++ b/.github/workflows/tests@v1.yml @@ -39,6 +39,9 @@ jobs: path: ~/.m2/repository key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ steps.get-pom-hash.outputs.value }} + - name: Install guava-shaded + run: mvn install -pl guava-shaded + - name: Compile source and tests run: mvn -B compile test-compile -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true @@ -113,6 +116,9 @@ jobs: path: ~/.m2/repository key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }} + - name: Install guava-shaded + run: mvn install -pl guava-shaded + - name: Run unit tests run: mvn -B test -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true From 823d4fee548eb4aa3331dd42eab75fc755a1105a Mon Sep 17 00:00:00 2001 From: Bouncheck <36934780+Bouncheck@users.noreply.github.com> Date: Mon, 17 Mar 2025 13:01:11 +0100 Subject: [PATCH 46/48] Remove exception catch in `prepared_stmt_metadata_update_loopholes_test` Since incorporating "JAVA-3057 Allow decoding a UDT that has more fields than expected" the asuumptions of removed check are no longer valid. --- .../driver/core/cql/BoundStatementCcmIT.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BoundStatementCcmIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BoundStatementCcmIT.java index 12159bb786c..4b5f3b28a44 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BoundStatementCcmIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/cql/BoundStatementCcmIT.java @@ -522,25 +522,15 @@ public void prepared_stmt_metadata_update_loopholes_test( } row = session.execute(stmtUDTTableWCS.bind(1)).one(); - try { - assertThat(row.getUdtValue(1).size()).isEqualTo(expectedColumnCount); - // if isPreparedStatementInvalidationBroken is true it should throw exception - assertThat(isPreparedStatementInvalidationBroken).isFalse(); - } catch (java.lang.IllegalArgumentException e) { - assertThat(isPreparedStatementInvalidationBroken).isTrue(); - } + + assertThat(row.getUdtValue(1).size()).isEqualTo(expectedColumnCount); assertThat(row.getColumnDefinitions().size()).isEqualTo(2); assertThat(getUDTColumnCount(row.getColumnDefinitions().get(1))) .isEqualTo(expectedColumnCount); row = session.execute(stmtUDTTableTS.bind(1)).one(); - try { - assertThat(row.getUdtValue(1).size()).isEqualTo(expectedColumnCount); - // if isPreparedStatementInvalidationBroken is true it should throw exception - assertThat(isPreparedStatementInvalidationBroken).isFalse(); - } catch (java.lang.IllegalArgumentException e) { - assertThat(isPreparedStatementInvalidationBroken).isTrue(); - } + + assertThat(row.getUdtValue(1).size()).isEqualTo(expectedColumnCount); assertThat(row.getColumnDefinitions().size()).isEqualTo(2); assertThat(getUDTColumnCount(row.getColumnDefinitions().get(1))) .isEqualTo(expectedColumnCount); From 8e243717ec39582f7f49fd3f70c66dea287714b4 Mon Sep 17 00:00:00 2001 From: Bouncheck <36934780+Bouncheck@users.noreply.github.com> Date: Mon, 17 Mar 2025 13:56:17 +0100 Subject: [PATCH 47/48] Switch shaded guava's groupId in osgi-tests Switches from `org.apache.cassandra` to `com.scylladb` in `BundleOptions#commonBundles()`. --- .../oss/driver/internal/osgi/support/BundleOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java index 3e6171ca530..713eb68f207 100644 --- a/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java +++ b/osgi-tests/src/test/java/com/datastax/oss/driver/internal/osgi/support/BundleOptions.java @@ -35,7 +35,7 @@ public class BundleOptions { public static CompositeOption commonBundles() { return () -> options( - mavenBundle("org.apache.cassandra", "java-driver-guava-shaded").versionAsInProject(), + mavenBundle("com.scylladb", "java-driver-guava-shaded").versionAsInProject(), mavenBundle("io.dropwizard.metrics", "metrics-core").versionAsInProject(), mavenBundle("org.slf4j", "slf4j-api").versionAsInProject(), mavenBundle("org.hdrhistogram", "HdrHistogram").versionAsInProject(), From 493b4a3680b00e5206f073acafb2d9f3cd36201b Mon Sep 17 00:00:00 2001 From: Bouncheck <36934780+Bouncheck@users.noreply.github.com> Date: Mon, 17 Mar 2025 14:01:29 +0100 Subject: [PATCH 48/48] Typo: increase line's loglevel in CcmBridge --- .../com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java index 035ed20fad0..9fdb79fd496 100644 --- a/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java +++ b/test-infra/src/main/java/com/datastax/oss/driver/api/testinfra/ccm/CcmBridge.java @@ -151,7 +151,7 @@ public class CcmBridge implements AutoCloseable { static { Map envMap = Maps.newHashMap(new ProcessBuilder().environment()); if (SCYLLA_ENABLEMENT) { - LOG.info("Overriding distribution variable because 'ccm.scylla = true' was passed"); + LOG.debug("Overriding distribution variable because 'ccm.scylla = true' was passed"); DISTRIBUTION = BackendType.SCYLLA; if (SCYLLA_ENTERPRISE) {