-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: regression in auth token configuration (#9)
* fix: regression in auth token configuration The regression was caused by masking the auth token so Kafka Connect would not log it. For the avoidance of doubts: This issue could not be abused to get unauthorized access to QuestDB.
- Loading branch information
Showing
6 changed files
with
303 additions
and
192 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
77 changes: 77 additions & 0 deletions
77
connector/src/test/java/io/questdb/kafka/ConnectTestUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| package io.questdb.kafka; | ||
|
|
||
| import org.apache.kafka.connect.json.JsonConverter; | ||
| import org.apache.kafka.connect.runtime.AbstractStatus; | ||
| import org.apache.kafka.connect.runtime.ConnectorConfig; | ||
| import org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo; | ||
| import org.apache.kafka.connect.storage.StringConverter; | ||
| import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; | ||
| import org.awaitility.Awaitility; | ||
| import org.testcontainers.containers.GenericContainer; | ||
|
|
||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.concurrent.atomic.AtomicInteger; | ||
|
|
||
| import static java.util.concurrent.TimeUnit.MILLISECONDS; | ||
| import static java.util.concurrent.TimeUnit.SECONDS; | ||
| import static org.apache.kafka.connect.runtime.ConnectorConfig.KEY_CONVERTER_CLASS_CONFIG; | ||
| import static org.apache.kafka.connect.runtime.ConnectorConfig.VALUE_CONVERTER_CLASS_CONFIG; | ||
| import static org.junit.jupiter.api.Assertions.fail; | ||
|
|
||
| public final class ConnectTestUtils { | ||
| public static final long CONNECTOR_START_TIMEOUT_MS = SECONDS.toMillis(60); | ||
| public static final String CONNECTOR_NAME = "questdb-sink-connector"; | ||
| private static final AtomicInteger ID_GEN = new AtomicInteger(0); | ||
|
|
||
| private ConnectTestUtils() { | ||
| } | ||
|
|
||
| static void assertConnectorTaskRunningEventually(EmbeddedConnectCluster connect) { | ||
| assertConnectorTaskStateEventually(connect, AbstractStatus.State.RUNNING); | ||
| } | ||
|
|
||
| static void assertConnectorTaskFailedEventually(EmbeddedConnectCluster connect) { | ||
| assertConnectorTaskStateEventually(connect, AbstractStatus.State.FAILED); | ||
| } | ||
|
|
||
| static void assertConnectorTaskStateEventually(EmbeddedConnectCluster connect, AbstractStatus.State expectedState) { | ||
| Awaitility.await().atMost(CONNECTOR_START_TIMEOUT_MS, MILLISECONDS).untilAsserted(() -> assertConnectorTaskState(connect, CONNECTOR_NAME, expectedState)); | ||
| } | ||
|
|
||
| static Map<String, String> baseConnectorProps(GenericContainer<?> questDBContainer, String topicName) { | ||
| Map<String, String> props = new HashMap<>(); | ||
| props.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, QuestDBSinkConnector.class.getName()); | ||
| props.put("topics", topicName); | ||
| props.put(KEY_CONVERTER_CLASS_CONFIG, StringConverter.class.getName()); | ||
| props.put(VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); | ||
| props.put("host", questDBContainer.getHost() + ":" + questDBContainer.getMappedPort(QuestDBUtils.QUESTDB_ILP_PORT)); | ||
| return props; | ||
| } | ||
|
|
||
| static void assertConnectorTaskState(EmbeddedConnectCluster connect, String connectorName, AbstractStatus.State expectedState) { | ||
| ConnectorStateInfo info = connect.connectorStatus(connectorName); | ||
| if (info == null) { | ||
| fail("Connector " + connectorName + " not found"); | ||
| } | ||
| List<ConnectorStateInfo.TaskState> taskStates = info.tasks(); | ||
| if (taskStates.size() == 0) { | ||
| fail("No tasks found for connector " + connectorName); | ||
| } | ||
| for (ConnectorStateInfo.TaskState taskState : taskStates) { | ||
| if (!Objects.equals(taskState.state(), expectedState.toString())) { | ||
| fail("Task " + taskState.id() + " for connector " + connectorName + " is in state " + taskState.state() + " but expected " + expectedState); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| static String newTopicName() { | ||
| return "topic" + ID_GEN.getAndIncrement(); | ||
| } | ||
|
|
||
| static String newTableName() { | ||
| return "table" + ID_GEN.getAndIncrement(); | ||
| } | ||
| } |
89 changes: 89 additions & 0 deletions
89
connector/src/test/java/io/questdb/kafka/QuestDBSinkConnectorEmbeddedAuthTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| package io.questdb.kafka; | ||
|
|
||
| import org.apache.kafka.connect.data.Schema; | ||
| import org.apache.kafka.connect.data.SchemaBuilder; | ||
| import org.apache.kafka.connect.data.Struct; | ||
| import org.apache.kafka.connect.json.JsonConverter; | ||
| import org.apache.kafka.connect.storage.Converter; | ||
| import org.apache.kafka.connect.storage.ConverterConfig; | ||
| import org.apache.kafka.connect.storage.ConverterType; | ||
| import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; | ||
| import org.junit.jupiter.api.BeforeEach; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.slf4j.LoggerFactory; | ||
| import org.testcontainers.containers.FixedHostPortGenericContainer; | ||
| import org.testcontainers.containers.GenericContainer; | ||
| import org.testcontainers.containers.output.Slf4jLogConsumer; | ||
| import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; | ||
| import org.testcontainers.junit.jupiter.Container; | ||
| import org.testcontainers.junit.jupiter.Testcontainers; | ||
| import org.testcontainers.utility.MountableFile; | ||
|
|
||
| import java.util.Map; | ||
|
|
||
| import static java.util.Collections.singletonMap; | ||
|
|
||
| @Testcontainers | ||
| public class QuestDBSinkConnectorEmbeddedAuthTest { | ||
| private EmbeddedConnectCluster connect; | ||
| private Converter converter; | ||
| private String topicName; | ||
|
|
||
| // must match the user in authDb.txt | ||
| private static final String TEST_USER_TOKEN = "UvuVb1USHGRRT08gEnwN2zGZrvM4MsLQ5brgF6SVkAw="; | ||
| private static final String TEST_USER_NAME = "testUser1"; | ||
|
|
||
| @Container | ||
| private static GenericContainer<?> questDBContainer = newQuestDbConnector(); | ||
|
|
||
| private static GenericContainer<?> newQuestDbConnector() { | ||
| FixedHostPortGenericContainer<?> container = new FixedHostPortGenericContainer<>("questdb/questdb:7.3"); | ||
| container.addExposedPort(QuestDBUtils.QUESTDB_HTTP_PORT); | ||
| container.addExposedPort(QuestDBUtils.QUESTDB_ILP_PORT); | ||
| container.setWaitStrategy(new LogMessageWaitStrategy().withRegEx(".*server-main enjoy.*")); | ||
| container.withCopyFileToContainer(MountableFile.forClasspathResource("/authDb.txt"), "/var/lib/questdb/conf/authDb.txt"); | ||
| container.withEnv("QDB_LINE_TCP_AUTH_DB_PATH", "conf/authDb.txt"); | ||
| return container.withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger("questdb"))); | ||
| } | ||
|
|
||
| @BeforeEach | ||
| public void setUp() { | ||
| topicName = ConnectTestUtils.newTopicName(); | ||
| JsonConverter jsonConverter = new JsonConverter(); | ||
| jsonConverter.configure(singletonMap(ConverterConfig.TYPE_CONFIG, ConverterType.VALUE.getName())); | ||
| converter = jsonConverter; | ||
|
|
||
| connect = new EmbeddedConnectCluster.Builder() | ||
| .name("questdb-connect-cluster") | ||
| .build(); | ||
|
|
||
| connect.start(); | ||
| } | ||
|
|
||
| @Test | ||
| public void testSmoke() { | ||
| connect.kafka().createTopic(topicName, 1); | ||
| Map<String, String> props = ConnectTestUtils.baseConnectorProps(questDBContainer, topicName); | ||
| props.put(QuestDBSinkConnectorConfig.USERNAME, TEST_USER_NAME); | ||
| props.put(QuestDBSinkConnectorConfig.TOKEN, TEST_USER_TOKEN); | ||
|
|
||
| connect.configureConnector(ConnectTestUtils.CONNECTOR_NAME, props); | ||
| ConnectTestUtils.assertConnectorTaskRunningEventually(connect); | ||
| Schema schema = SchemaBuilder.struct().name("com.example.Person") | ||
| .field("firstname", Schema.STRING_SCHEMA) | ||
| .field("lastname", Schema.STRING_SCHEMA) | ||
| .field("age", Schema.INT8_SCHEMA) | ||
| .build(); | ||
|
|
||
| Struct struct = new Struct(schema) | ||
| .put("firstname", "John") | ||
| .put("lastname", "Doe") | ||
| .put("age", (byte) 42); | ||
|
|
||
| connect.kafka().produce(topicName, "key", new String(converter.fromConnectData(topicName, schema, struct))); | ||
|
|
||
| QuestDBUtils.assertSqlEventually(questDBContainer, "\"firstname\",\"lastname\",\"age\"\r\n" | ||
| + "\"John\",\"Doe\",42\r\n", | ||
| "select firstname,lastname,age from " + topicName); | ||
| } | ||
| } |
Oops, something went wrong.