From 72ffbca653fcd642fcf9d7416b59a6e5fbbfb679 Mon Sep 17 00:00:00 2001 From: Alka Trivedi Date: Wed, 25 Oct 2023 12:31:26 +0530 Subject: [PATCH 1/2] testing code for benchmarking long running session removals Committer: Alka Trivedi On branch benchmarking Changes to be committed: new file: samples/samples/benchmarking.py --- samples/samples/benchmarking.py | 126 ++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 samples/samples/benchmarking.py diff --git a/samples/samples/benchmarking.py b/samples/samples/benchmarking.py new file mode 100644 index 0000000000..14251654e6 --- /dev/null +++ b/samples/samples/benchmarking.py @@ -0,0 +1,126 @@ +import threading +from google.cloud import spanner_v1 +import time +import math + +from google.cloud.spanner_v1 import transaction, pool, _helpers +from google.cloud import spanner +import concurrent.futures + +# Define your Spanner instance and database information +project_id = "your_project_id" +instance_id = "your_instance_id" +database_id = "your_database_id" +spanner_client = spanner_v1.Client(project=project_id) + +# Create a Spanner database instance +instance = spanner_client.instance(instance_id) +pool = pool.FixedSizePool(size = 10, logging_enabled=True) +database = instance.database(pool=pool, database_id=database_id, close_inactive_transactions=True) + +transaction_time = [] + + +def calculatePercentile(latencies): + # sort the latencies array + latencies.sort() + + # calculate p50 (50th percentile) + p50Index = math.floor(0.5*len(latencies)) + p50Latency = latencies[p50Index] + + # calculate p90 (90th percentile) + p90Index = math.floor(0.9*len(latencies)) + p90Latency = latencies[p90Index] + + return [p50Latency, p90Latency] + +# [START spanner_query_data] +def query_data(thread_id): + print("running thread ", thread_id) + start_time = time.time() + time.sleep(10) + with database.snapshot() as snapshot: + results = snapshot.execute_sql( + "SELECT 1 FROM Singers" + ) + + # for row in results: + # print(row) + + # for row in results: + # print("SingerId: {}, FirstName: {}, LastName: {}".format(*row)) + + end_time = time.time() + transaction_time.append(end_time-start_time) +# [END spanner_query_data] + +# [START spanner_batch_transaction] +def batch_transaction(thread_id): + print("running thread ", thread_id) + start_time = time.time() + time.sleep(10) + batch_txn = database.batch_snapshot() + batches = batch_txn.execute_sql( + 'SELECT * FROM Singers', + ) + results = [] + for batch in batches: + results.append("SingerId: {}, FirstName: {}, LastName: {}".format(*batch)) + + # for batch in batches: + # for row in batch_txn.process_read_batch(batch): + # results.append("SingerId: {}, FirstName: {}, LastName: {}".format(*row)) + # for result in results: + # print(result) + + end_time = time.time() + transaction_time.append(end_time-start_time) +# [END spanner_batch_transaction] + +# [START insert_with_dml] +def insert_with_dml(i): + """Inserts data with a DML statement into the database.""" + print("running thread ", i) + start_time = time.time() + time.sleep(10) + + def insert_singers(transaction): + row_ct = transaction.execute_update( + "INSERT Singers (SingerId, FirstName, LastName) VALUES ({}, 'Google{}', 'India{}')".format(i, i, i) + ) + print("{} record(s) inserted.".format(row_ct)) + + database.run_in_transaction(insert_singers) + end_time = time.time() + transaction_time.append(end_time-start_time) +# [END insert_with_dml] + +# Define the number of threads +num_threads = 20 +starting = 1 + +# Create and start the threads +threads = [] +start = time.time() +for i in range(starting,starting+num_threads): + thread = threading.Thread(target=query_data, args=(i,)) + thread.start() + threads.append(thread) + + +# Wait for all threads to complete +for thread in threads: + thread.join() + +print("All threads have completed.") +end = time.time() + +# Print the total execution time +print("total time taken by the execution: ", end-start) +for t in transaction_time: + with open ('output.txt', 'a') as file: + file.write(str(t)+"\n") +# latency = calculatePercentile(transaction_time) +# print("p50 latency is: ", latency[0]) +# print("p90 latency is: ", latency[1]) \ No newline at end of file From f5b1b34fcc9427a5015a61cd0bdc6a16e97671ef Mon Sep 17 00:00:00 2001 From: Alka Trivedi Date: Wed, 25 Oct 2023 12:41:40 +0530 Subject: [PATCH 2/2] test code snippet for benchmarking long running session removal --- samples/samples/benchmarking.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/samples/benchmarking.py b/samples/samples/benchmarking.py index 14251654e6..f0c6f2f09a 100644 --- a/samples/samples/benchmarking.py +++ b/samples/samples/benchmarking.py @@ -118,6 +118,8 @@ def insert_singers(transaction): # Print the total execution time print("total time taken by the execution: ", end-start) + +#Writing transaction time to an output file for t in transaction_time: with open ('output.txt', 'a') as file: file.write(str(t)+"\n")