diff --git a/scripts/donations-new-table-reload.py b/scripts/donations-new-table-reload.py new file mode 100644 index 0000000..3a46061 --- /dev/null +++ b/scripts/donations-new-table-reload.py @@ -0,0 +1,92 @@ +""" +Module to read data from DynamoDB tables and populate into a single DynamoDB table, with logging and dry-run feature. + +This module provides functions to read items from multiple DynamoDB tables that were storing donations data for +individual funds, and populate these items into a single DynamoDB table that will hold donations. +""" + +import sys +import boto3 +from botocore.exceptions import ClientError +import logging + +def get_dynamodb_tables(dynamodb_client, exclude_table): + """ + Get a list of DynamoDB table names, excluding a specific table. + + Args: + dynamodb_client: A boto3 DynamoDB client. + exclude_table: The name of the table to exclude from the list. + + Returns: + A list of table names. + """ + try: + response = dynamodb_client.list_tables() + return [table for table in response['TableNames'] if table != exclude_table] + except ClientError as error: + logging.error(f"Error fetching table names: {error}") + return [] + +def read_table_items(dynamodb_client, table_name): + """ + Read all items from a DynamoDB table. + + Args: + dynamodb_client: A boto3 DynamoDB client. + table_name: The name of the DynamoDB table. + + Returns: + A list of items from the table. + """ + try: + response = dynamodb_client.scan(TableName=table_name) + items = response['Items'] + logging.info(f"Read {len(items)} items from table '{table_name}'") + return items + except ClientError as error: + logging.error(f"Error reading items from table {table_name}: {error}") + return [] + +def write_items_to_table(dynamodb_client, target_table, items, original_table_name, dry_run): + """ + Write items to a DynamoDB table with an additional attribute. + + Args: + dynamodb_client: A boto3 DynamoDB client. + target_table: The name of the target DynamoDB table. + items: A list of items to write. + original_table_name: The name of the original table to use for the 'fund_id' attribute. + dry_run: If True, do not write to the table, only log the actions. + """ + for item in items: + item['fund_id'] = {'S': original_table_name} + if not dry_run: + try: + dynamodb_client.put_item(TableName=target_table, Item=item) + except ClientError as error: + logging.error(f"Error writing item to table {target_table}: {error}") + +def main(dry_run=False, region_name='us-east-1'): + """ + Main function to migrate DynamoDB items from multiple tables to a single table. + + Args: + dry_run: If True, do not write to the target table, only read and log the actions. + region_name: AWS region where the DynamoDB tables are located. + """ + dynamodb_client = boto3.client('dynamodb', region_name=region_name) + source_tables = get_dynamodb_tables(dynamodb_client, 'funds') + target_table = 'donations-all' + + for table_name in source_tables: + items = read_table_items(dynamodb_client, table_name) + write_items_to_table(dynamodb_client, target_table, items, table_name, dry_run) + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s [%(levelname)s] %(message)s', + stream=sys.stdout + ) + main(dry_run=True, region_name='us-east-1') # Set to False to execute data writing diff --git a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/AbstractFundOperationsTestCommon.java b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/AbstractFundOperationsTestCommon.java index f51a089..3431a0d 100644 --- a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/AbstractFundOperationsTestCommon.java +++ b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/AbstractFundOperationsTestCommon.java @@ -1,9 +1,10 @@ package com.yuriytkach.tracker.fundraiser; +import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.DONATIONS_TABLE; import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUND; import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUNDS_TABLE; -import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUND_1_TABLE; -import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.ALL_ATTRIBUTES; +import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.ALL_ATTRIBUTES_WITHOUT_FUND_ID; +import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_FUND_ID; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_ID; import static org.assertj.core.api.Assertions.assertThat; @@ -47,21 +48,23 @@ public abstract class AbstractFundOperationsTestCommon { @AfterEach void cleanUp() { log.info("----- CLEANUP ------"); - deleteItemByIdDirectly(FUND_1_TABLE, COL_ID, ITEM_ID_1, ITEM_ID_2); + deleteItemByIdDirectly(DONATIONS_TABLE, COL_ID, ITEM_ID_1, ITEM_ID_2); deleteItemByIdDirectly(FUNDS_TABLE, DynamoDbFundStorageClient.COL_NAME, FUND_2_NAME); deleteTableIfExists(FundService.FUND_TABLE_PREFIX + FUND_2_NAME); fundStorageClient.save(FUND); } - protected Optional getDonationDirectlyById(final String donationId) { + protected Optional getDonationDirectlyById(final String donationId) { final GetItemRequest dbGetItemRequest = GetItemRequest.builder() - .tableName(FUND_1_TABLE) + .tableName(DynamoDbTestResource.DONATIONS_TABLE) .key(Map.of(COL_ID, AttributeValue.builder().s(donationId).build())) - .attributesToGet(ALL_ATTRIBUTES) + .attributesToGet(StreamEx.of(ALL_ATTRIBUTES_WITHOUT_FUND_ID, 0, ALL_ATTRIBUTES_WITHOUT_FUND_ID.length) + .append(COL_FUND_ID).toArray(String.class)) .build(); final GetItemResponse response = dynamoDB.getItem(dbGetItemRequest); assertThat(response.item()).isNotEmpty(); - return DynamoDbDonationClientDonation.parseDonation(response.item()); + return DynamoDbDonationClientDonation.parseDonation(response.item()) + .map(donation -> new DonationWithFundId(response.item().get(COL_FUND_ID).s(), donation)); } protected Optional getFundDirectlyByName(final String name) { @@ -75,7 +78,7 @@ protected Optional getFundDirectlyByName(final String name) { return DynamoDbFundStorageClient.parseFund(response.item()); } - protected void deleteItemByIdDirectly(final String fundTable, final String keyColumn, final Object... ids) { + protected void deleteItemByIdDirectly(final String table, final String keyColumn, final Object... ids) { final var requests = StreamEx.of(ids) .map(id -> AttributeValue.builder().s(id.toString()).build()) .map(attrValue -> Map.of(keyColumn, attrValue)) @@ -84,7 +87,7 @@ protected void deleteItemByIdDirectly(final String fundTable, final String keyCo .toList(); dynamoDB.batchWriteItem(BatchWriteItemRequest.builder() - .requestItems(Map.of(fundTable, requests)) + .requestItems(Map.of(table, requests)) .build()); } @@ -95,4 +98,6 @@ private void deleteTableIfExists(final String tableName) { } } + public record DonationWithFundId(String fundId, Donation donation) { } + } diff --git a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/CmdTrackAwsLambdaIT.java b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/CmdTrackAwsLambdaIT.java index dd5c92d..07b561e 100644 --- a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/CmdTrackAwsLambdaIT.java +++ b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/CmdTrackAwsLambdaIT.java @@ -1,11 +1,11 @@ package com.yuriytkach.tracker.fundraiser; import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUND; -import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUNDS_TABLE; -import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUND_1_TABLE; +import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUND_1_ID; import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUND_OWNER; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_AMOUNT; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_CURR; +import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_FUND_ID; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_ID; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_PERSON; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_TIME; @@ -55,7 +55,6 @@ import one.util.streamex.StreamEx; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.ListTablesResponse; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; @Slf4j @@ -145,11 +144,6 @@ void shouldCreateFund(final String cmdTextSuffix, final String expectedDesc, fin assertThat(fund2.getCurrency()).isEqualTo(fund2Currency); assertThat(fund2.getCreatedAt()).isCloseTo(Instant.now(), within(1, SECONDS)); assertThat(fund2.getUpdatedAt()).isCloseTo(Instant.now(), within(1, SECONDS)); - - final ListTablesResponse allTablesResponse = dynamoDB.listTables(); - assertThat(allTablesResponse.tableNames()).containsExactlyInAnyOrder( - expectedFund2TableName, FUNDS_TABLE, FUND_1_TABLE - ); } @Test @@ -190,11 +184,6 @@ void shouldDeleteFund() { .responseType(SlackResponse.RESPONSE_PRIVATE) .text(":white_check_mark: Deleted fund `" + FUND_2_NAME + "`") .build())); - - final ListTablesResponse allTablesResponse = dynamoDB.listTables(); - assertThat(allTablesResponse.tableNames()).containsExactlyInAnyOrder( - FUNDS_TABLE, FUND_1_TABLE - ); } @ParameterizedTest @@ -225,14 +214,17 @@ void shouldReturnOKIfTrackSuccessful(final String person, final String expectedP + " - :open_book: `fundy` 22.30% [223 of 1000] EUR - :bank:-1") .build())); - final Optional donation = getDonationDirectlyById(ITEM_ID_1.toString()); - assertThat(donation).hasValue(Donation.builder() - .id(ITEM_ID_1.toString()) - .currency(FUND.getCurrency()) - .amount(123) - .dateTime(Instant.parse("2022-02-01T12:13:00Z")) - .person(expectedPerson) - .build()); + final Optional donation = getDonationDirectlyById(ITEM_ID_1.toString()); + assertThat(donation).hasValue(new DonationWithFundId( + FUND_1_ID, + Donation.builder() + .id(ITEM_ID_1.toString()) + .currency(FUND.getCurrency()) + .amount(123) + .dateTime(Instant.parse("2022-02-01T12:13:00Z")) + .person(expectedPerson) + .build() + )); final Optional fund = getFundDirectlyByName(FUND.getName()); assertThat(fund).hasValue(FUND.toBuilder() @@ -417,13 +409,14 @@ private void addDonationDirectly( ) { final Map item = new HashMap<>(); item.put(COL_ID, AttributeValue.builder().s(itemId.toString()).build()); + item.put(COL_FUND_ID, AttributeValue.builder().s(FUND_1_ID).build()); item.put(COL_CURR, AttributeValue.builder().s(curr).build()); item.put(COL_PERSON, AttributeValue.builder().s(person).build()); item.put(COL_AMOUNT, AttributeValue.builder().n(String.valueOf(amount)).build()); item.put(COL_TIME, AttributeValue.builder().s(dateTime.toString()).build()); final var putRequest = PutItemRequest.builder() - .tableName(FUND_1_TABLE) + .tableName(DynamoDbTestResource.DONATIONS_TABLE) .item(item) .build(); dynamoDB.putItem(putRequest); diff --git a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/DynamoDbTestResource.java b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/DynamoDbTestResource.java index fc0764e..00a584e 100644 --- a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/DynamoDbTestResource.java +++ b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/DynamoDbTestResource.java @@ -39,9 +39,11 @@ public class DynamoDbTestResource implements QuarkusTestResourceLifecycleManager { public static final String FUNDS_TABLE = "all-funds-table"; - public static final String ENABLED_INDEX = "test-mono-index"; - public static final String FUND_1_TABLE = "donations-table"; - public static final String FUND_2_TABLE = "disabled-table"; + public static final String ENABLED_INDEX = "test-enabled-index"; + public static final String FUND_ID_INDEX = "test-fund-id-index"; + public static final String DONATIONS_TABLE = "donations-table"; + public static final String FUND_1_ID = "fund-id-1"; + public static final String FUND_2_ID = "fund-id-2"; public static final String FUND_OWNER = "owner"; public static final String FUND_RED = "red"; public static final String FUND_DESC = "description"; @@ -51,7 +53,7 @@ public class DynamoDbTestResource implements QuarkusTestResourceLifecycleManager public static final String FUND_DISABLED_NAME = "dis-fund"; public static final Fund FUND = Fund.builder() - .id(FUND_1_TABLE) + .id(FUND_1_ID) .enabled(true) .name(FUND_1_NAME) .goal(1000) @@ -66,7 +68,7 @@ public class DynamoDbTestResource implements QuarkusTestResourceLifecycleManager .build(); public static final Fund FUND_DISABLED = Fund.builder() - .id(FUND_2_TABLE) + .id(FUND_2_ID) .enabled(false) .name(FUND_DISABLED_NAME) .goal(2000) @@ -99,9 +101,16 @@ public Map start() { dynamoDB, FUNDS_TABLE, DynamoDbFundStorageClient.COL_NAME, - buildSecondaryIndex(ENABLED_INDEX, DynamoDbFundStorageClient.COL_ENABLED) + buildSecondaryIndex(ENABLED_INDEX, DynamoDbFundStorageClient.COL_ENABLED), + ScalarAttributeType.N + ); + createTable( + dynamoDB, + DONATIONS_TABLE, + DynamoDbDonationClientDonation.COL_ID, + buildSecondaryIndex(FUND_ID_INDEX, DynamoDbDonationClientDonation.COL_FUND_ID), + ScalarAttributeType.S ); - createTable(dynamoDB, FUND_1_TABLE, DynamoDbDonationClientDonation.COL_ID, null); createFundItem(dynamoDB, FUND); createFundItem(dynamoDB, FUND_DISABLED); @@ -109,6 +118,8 @@ public Map start() { return Map.of( "app.funds-table", FUNDS_TABLE, "app.funds-enabled-index", ENABLED_INDEX, + "app.donations-table", DONATIONS_TABLE, + "app.donations-fund-id-index", FUND_ID_INDEX, "quarkus.dynamodb.endpoint-override", url ); } @@ -166,7 +177,8 @@ private void createTable( final AmazonDynamoDB dynamoDB, final String tableName, final String keyColumn, - @Nullable final GlobalSecondaryIndex index + @Nullable final GlobalSecondaryIndex index, + final ScalarAttributeType secondKeyType ) { final CreateTableRequest request = new CreateTableRequest(); request.setTableName(tableName); @@ -181,7 +193,7 @@ private void createTable( if (index != null) { request.withGlobalSecondaryIndexes(List.of(index)); request.withAttributeDefinitions(new AttributeDefinition( - index.getKeySchema().get(0).getAttributeName(), ScalarAttributeType.N)); + index.getKeySchema().get(0).getAttributeName(), secondKeyType)); } final CreateTableResult table = dynamoDB.createTable(request); diff --git a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/FundStatusAwsLambdaIT.java b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/FundStatusAwsLambdaIT.java index f1ac4a5..cb929dc 100644 --- a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/FundStatusAwsLambdaIT.java +++ b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/FundStatusAwsLambdaIT.java @@ -2,6 +2,7 @@ import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_AMOUNT; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_CURR; +import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_FUND_ID; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_ID; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_PERSON; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_TIME; @@ -261,13 +262,14 @@ private void addDonationDirectly( ) { final Map item = new HashMap<>(); item.put(COL_ID, AttributeValue.builder().s(itemId.toString()).build()); + item.put(COL_FUND_ID, AttributeValue.builder().s(DynamoDbTestResource.FUND_1_ID).build()); item.put(COL_CURR, AttributeValue.builder().s(curr).build()); item.put(COL_PERSON, AttributeValue.builder().s(person).build()); item.put(COL_AMOUNT, AttributeValue.builder().n(String.valueOf(amount)).build()); item.put(COL_TIME, AttributeValue.builder().s(dateTime.toString()).build()); final var putRequest = PutItemRequest.builder() - .tableName(DynamoDbTestResource.FUND_1_TABLE) + .tableName(DynamoDbTestResource.DONATIONS_TABLE) .item(item) .build(); dynamoDB.putItem(putRequest); @@ -282,13 +284,13 @@ private void deleteItemByIdDirectly(final UUID... ids) { .toList(); dynamoDB.batchWriteItem(BatchWriteItemRequest.builder() - .requestItems(Map.of(DynamoDbTestResource.FUND_1_TABLE, requests)) + .requestItems(Map.of(DynamoDbTestResource.DONATIONS_TABLE, requests)) .build()); } private Fund dummyFund() { return Fund.builder() - .id(DynamoDbTestResource.FUND_1_TABLE) + .id(DynamoDbTestResource.FUND_1_ID) .goal(1000) .currency(FUND_CURR) .raised(0) diff --git a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/mono/MonobankHookIT.java b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/mono/MonobankHookIT.java index 51efc5c..37f5113 100644 --- a/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/mono/MonobankHookIT.java +++ b/src/integrationTest/java/com/yuriytkach/tracker/fundraiser/mono/MonobankHookIT.java @@ -1,7 +1,8 @@ package com.yuriytkach.tracker.fundraiser.mono; +import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.DONATIONS_TABLE; import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUND; -import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUND_1_TABLE; +import static com.yuriytkach.tracker.fundraiser.DynamoDbTestResource.FUND_1_ID; import static com.yuriytkach.tracker.fundraiser.service.dynamodb.DynamoDbDonationClientDonation.COL_ID; import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThat; @@ -44,7 +45,7 @@ class MonobankHookIT extends AbstractFundOperationsTestCommon implements AwsLamb @AfterEach void cleanUpDataFromMono() { - deleteItemByIdDirectly(FUND_1_TABLE, COL_ID, MONO_STATEMENT_ID); + deleteItemByIdDirectly(DONATIONS_TABLE, COL_ID, MONO_STATEMENT_ID); } @Test @@ -75,14 +76,17 @@ void shouldTrackDonation() { .body("statusCode", equalTo(200)) .body("body", nullValue()); - final Optional donation = getDonationDirectlyById(MONO_STATEMENT_ID); - assertThat(donation).hasValue(Donation.builder() - .id(MONO_STATEMENT_ID) - .currency(Currency.EUR) - .amount(MONO_STATEMENT_AMOUNT) - .dateTime(MONO_STATEMENT_TIME) - .person("YuriyT") - .build()); + final Optional donation = getDonationDirectlyById(MONO_STATEMENT_ID); + assertThat(donation).hasValue(new DonationWithFundId( + FUND_1_ID, + Donation.builder() + .id(MONO_STATEMENT_ID) + .currency(Currency.EUR) + .amount(MONO_STATEMENT_AMOUNT) + .dateTime(MONO_STATEMENT_TIME) + .person("YuriyT") + .build() + )); final Optional fund = getFundDirectlyByName(FUND.getName()); assertThat(fund).hasValue(FUND.toBuilder() diff --git a/src/main/java/com/yuriytkach/tracker/fundraiser/config/FundTrackerConfig.java b/src/main/java/com/yuriytkach/tracker/fundraiser/config/FundTrackerConfig.java index 7d3e4e6..98d7e42 100644 --- a/src/main/java/com/yuriytkach/tracker/fundraiser/config/FundTrackerConfig.java +++ b/src/main/java/com/yuriytkach/tracker/fundraiser/config/FundTrackerConfig.java @@ -9,6 +9,10 @@ public interface FundTrackerConfig { String fundsEnabledIndex(); + String donationsTable(); + + String donationsFundIdIndex(); + String defaultFundColor(); String defaultPersonName(); diff --git a/src/main/java/com/yuriytkach/tracker/fundraiser/service/DonationStorageClient.java b/src/main/java/com/yuriytkach/tracker/fundraiser/service/DonationStorageClient.java index 1978578..6c05ffd 100644 --- a/src/main/java/com/yuriytkach/tracker/fundraiser/service/DonationStorageClient.java +++ b/src/main/java/com/yuriytkach/tracker/fundraiser/service/DonationStorageClient.java @@ -5,7 +5,7 @@ import com.yuriytkach.tracker.fundraiser.model.Donation; public interface DonationStorageClient { - void addAll(String id, Collection donations); + void addAll(String fundId, Collection donations); Collection findAll(String fundId); diff --git a/src/main/java/com/yuriytkach/tracker/fundraiser/service/dynamodb/DynamoDbDonationClientDonation.java b/src/main/java/com/yuriytkach/tracker/fundraiser/service/dynamodb/DynamoDbDonationClientDonation.java index 8f8e472..3e51945 100644 --- a/src/main/java/com/yuriytkach/tracker/fundraiser/service/dynamodb/DynamoDbDonationClientDonation.java +++ b/src/main/java/com/yuriytkach/tracker/fundraiser/service/dynamodb/DynamoDbDonationClientDonation.java @@ -5,8 +5,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; +import com.yuriytkach.tracker.fundraiser.config.FundTrackerConfig; import com.yuriytkach.tracker.fundraiser.model.Currency; import com.yuriytkach.tracker.fundraiser.model.Donation; import com.yuriytkach.tracker.fundraiser.service.DonationStorageClient; @@ -19,8 +19,11 @@ import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemRequest; import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse; +import software.amazon.awssdk.services.dynamodb.model.ComparisonOperator; +import software.amazon.awssdk.services.dynamodb.model.Condition; import software.amazon.awssdk.services.dynamodb.model.PutRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanRequest; +import software.amazon.awssdk.services.dynamodb.model.QueryRequest; +import software.amazon.awssdk.services.dynamodb.model.QueryResponse; import software.amazon.awssdk.services.dynamodb.model.WriteRequest; @Slf4j @@ -33,8 +36,9 @@ public class DynamoDbDonationClientDonation implements DonationStorageClient { public static final String COL_TIME = "time"; public static final String COL_PERSON = "person"; public static final String COL_ID = "id"; + public static final String COL_FUND_ID = "find_id"; - public static final String[] ALL_ATTRIBUTES = new String[] { + public static final String[] ALL_ATTRIBUTES_WITHOUT_FUND_ID = new String[] { COL_ID, COL_CURR, COL_AMOUNT, @@ -43,6 +47,7 @@ public class DynamoDbDonationClientDonation implements DonationStorageClient { }; private final DynamoDbClient dynamoDB; + private final FundTrackerConfig config; public static Optional parseDonation(final Map item) { if (item == null || item.isEmpty()) { @@ -60,14 +65,14 @@ public static Optional parseDonation(final Map @Override public void addAll(final String fundId, final Collection donations) { - log.debug("Saving donations to table `{}`: {}", fundId, donations.size()); + log.debug("Saving donations to fund `{}`: {}", fundId, donations.size()); final var writeRequests = StreamEx.of(donations) - .map(this::createPutRequest) + .map(donation -> createPutRequest(fundId, donation)) .map(putRequest -> WriteRequest.builder().putRequest(putRequest).build()) .toImmutableSet(); final BatchWriteItemRequest request = BatchWriteItemRequest.builder() - .requestItems(Map.of(fundId, writeRequests)) + .requestItems(Map.of(config.donationsTable(), writeRequests)) .build(); final BatchWriteItemResponse response = dynamoDB.batchWriteItem(request); log.debug("Saved donations. Consumed capacity: {}", response.consumedCapacity()); @@ -75,20 +80,33 @@ public void addAll(final String fundId, final Collection donations) { @Override public Collection findAll(final String fundId) { - final var request = ScanRequest.builder() - .tableName(fundId) - .attributesToGet(ALL_ATTRIBUTES) + final QueryRequest queryRequest = QueryRequest.builder() + .tableName(config.donationsTable()) + .indexName(config.donationsFundIdIndex()) + .keyConditions(Map.of(COL_FUND_ID, Condition.builder() + .attributeValueList(AttributeValue.fromS(fundId)) + .comparisonOperator(ComparisonOperator.EQ) + .build())) + .attributesToGet(ALL_ATTRIBUTES_WITHOUT_FUND_ID) .build(); - return dynamoDB.scanPaginator(request).items().stream() + final var donationsOpt = Optional.ofNullable(dynamoDB.query(queryRequest)) + .filter(QueryResponse::hasItems) + .map(QueryResponse::items); + + donationsOpt.ifPresent(list -> log.debug("Found donations for fund '{}': {}", fundId, list.size())); + + return donationsOpt.stream() + .flatMap(Collection::stream) .map(DynamoDbDonationClientDonation::parseDonation) .flatMap(Optional::stream) - .collect(Collectors.toUnmodifiableList()); + .toList(); } - private PutRequest createPutRequest(final Donation donation) { + private PutRequest createPutRequest(final String fundId, final Donation donation) { final Map item = new HashMap<>(); item.put(COL_ID, AttributeValue.builder().s(donation.getId()).build()); + item.put(COL_FUND_ID, AttributeValue.builder().s(fundId).build()); item.put(COL_CURR, AttributeValue.builder().s(donation.getCurrency().name()).build()); item.put(COL_AMOUNT, AttributeValue.builder().n(String.valueOf(donation.getAmount())).build()); item.put(COL_TIME, AttributeValue.builder().s(donation.getDateTime().toString()).build()); diff --git a/src/main/java/com/yuriytkach/tracker/fundraiser/service/dynamodb/DynamoDbFundStorageClient.java b/src/main/java/com/yuriytkach/tracker/fundraiser/service/dynamodb/DynamoDbFundStorageClient.java index 1cab8cb..52981e1 100644 --- a/src/main/java/com/yuriytkach/tracker/fundraiser/service/dynamodb/DynamoDbFundStorageClient.java +++ b/src/main/java/com/yuriytkach/tracker/fundraiser/service/dynamodb/DynamoDbFundStorageClient.java @@ -18,23 +18,15 @@ import lombok.extern.slf4j.Slf4j; import one.util.streamex.EntryStream; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.ComparisonOperator; import software.amazon.awssdk.services.dynamodb.model.Condition; -import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest; -import software.amazon.awssdk.services.dynamodb.model.CreateTableResponse; import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest; -import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest; import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; -import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; -import software.amazon.awssdk.services.dynamodb.model.KeyType; -import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; import software.amazon.awssdk.services.dynamodb.model.QueryRequest; import software.amazon.awssdk.services.dynamodb.model.QueryResponse; -import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; import software.amazon.awssdk.services.dynamodb.model.ScanRequest; @Slf4j @@ -97,27 +89,6 @@ public static Optional parseFund(final Map item) { @Override public void create(final Fund fund) { - final var createTableRequest = CreateTableRequest.builder() - .tableName(fund.getId()) - .keySchema( - KeySchemaElement.builder() - .keyType(KeyType.HASH) - .attributeName(DynamoDbDonationClientDonation.COL_ID) - .build() - ) - .attributeDefinitions( - AttributeDefinition.builder() - .attributeName(DynamoDbDonationClientDonation.COL_ID) - .attributeType(ScalarAttributeType.S) - .build() - ) - .provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(1L).writeCapacityUnits(1L).build()) - .build(); - - log.debug("Creating table for fund '{}': {}", fund.getName(), fund.getId()); - final CreateTableResponse createTableResponse = dynamoDB.createTable(createTableRequest); - log.info("Created table: {}", createTableResponse.tableDescription().tableName()); - save(fund); } @@ -225,13 +196,6 @@ public void remove(final Fund fund) { dynamoDB.deleteItem(itemDeleteRequest); log.info("Deleted fund record: {}", fund.getName()); - - final DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName(fund.getId()) - .build(); - dynamoDB.deleteTable(deleteTableRequest); - - log.info("Deleted fund table: {}", fund.getId()); } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 956202a..46b08da 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,6 +1,8 @@ app: funds-table: funds funds-enabled-index: enabled-index + donations-table: donations + donations-fund-id-index: fund-id-index default-fund-color: green default-person-name: noname funders: