From 5e47778a60e87c7d5f355aa6286e392606092687 Mon Sep 17 00:00:00 2001 From: baunsgaard Date: Tue, 1 Aug 2023 14:03:44 +0200 Subject: [PATCH] more --- .../org/apache/sysds/performance/Main.java | 4 +- .../org/apache/sysds/performance/Util.java | 33 +++-- .../performance/compression/APerfTest.java | 12 +- .../performance/compression/WriteTest.java | 124 +++++++++++++----- 4 files changed, 126 insertions(+), 47 deletions(-) diff --git a/src/test/java/org/apache/sysds/performance/Main.java b/src/test/java/org/apache/sysds/performance/Main.java index 67d05d0298a..a9a075b6122 100644 --- a/src/test/java/org/apache/sysds/performance/Main.java +++ b/src/test/java/org/apache/sysds/performance/Main.java @@ -70,7 +70,9 @@ private static void exec(int prog, String[] args) throws InterruptedException, E int cols = Integer.parseInt(args[2]); int unique = Integer.parseInt(args[3]); double sparsity = Double.parseDouble(args[4]); - new WriteTest(100, new ConstMatrix(rows, cols, unique, sparsity)).run(); + int k = Integer.parseInt(args[5]); + int n = Integer.parseInt(args[6]); + new WriteTest(n, new ConstMatrix(rows, cols, unique, sparsity), k).run(); default: break; } diff --git a/src/test/java/org/apache/sysds/performance/Util.java b/src/test/java/org/apache/sysds/performance/Util.java index d064356eba9..c7f0e5ad21b 100644 --- a/src/test/java/org/apache/sysds/performance/Util.java +++ b/src/test/java/org/apache/sysds/performance/Util.java @@ -25,6 +25,8 @@ import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing; public interface Util { + + public static double time(F f) { Timing time = new Timing(true); f.run(); @@ -55,17 +57,32 @@ public static double[] time(F f, int rep, IGenerate bq) throws InterruptedExc } public static String stats(double[] v) { - + final int l = v.length ; + final int remove = (int)Math.floor((double)l * 0.05); Arrays.sort(v); - final int l = v.length; - double min = v[0]; - double max = v[l - 1]; - double q25 = v[(int) (l / 4)]; - double q50 = v[(int) (l / 2)]; - double q75 = v[(int) ((l / 4) * 3)]; + double total = 0; + final int el = v.length - remove *2; + for(int i = remove; i < l-remove; i++) + total += v[i]; + + double mean = total / el; + + double var = 0; + for(int i = remove; i < l-remove; i++) + var += Math.pow(Math.abs(v[i] - mean), 2); + + double std = Math.sqrt(var / el); + + return String.format("%8.3f+-%6.3f ms", mean, std); + + // double min = v[0]; + // double max = v[l - 1]; + // double q25 = v[(int) (l / 4)]; + // double q50 = v[(int) (l / 2)]; + // double q75 = v[(int) ((l / 4) * 3)]; - return String.format("[%8.3f, %8.3f, %8.3f, %8.3f, %8.3f]", min, q25, q50, q75, max); + // return String.format("[%8.3f, %8.3f, %8.3f, %8.3f, %8.3f]", min, q25, q50, q75, max); } interface F { diff --git a/src/test/java/org/apache/sysds/performance/compression/APerfTest.java b/src/test/java/org/apache/sysds/performance/compression/APerfTest.java index 71ce9deda42..d319ccbf6bb 100644 --- a/src/test/java/org/apache/sysds/performance/compression/APerfTest.java +++ b/src/test/java/org/apache/sysds/performance/compression/APerfTest.java @@ -43,17 +43,25 @@ protected APerfTest(int N, IGenerate gen) { } protected void execute(F f, String name) throws InterruptedException { + warmup(f, 10); gen.generate(N); ret.clear(); double[] times = Util.time(f, N, gen); - System.out.println(String.format("%35s, %50s, %10s", name, Util.stats(times), makeResString(times))); + String retS = makeResString(times); + System.out.println(String.format("%35s, %s, %10s", name, Util.stats(times), retS)); + } + + protected void warmup(F f, int n) throws InterruptedException { + gen.generate(N); + ret.clear(); } protected void execute(F f, String name, int N) throws InterruptedException { gen.generate(N); ret.clear(); double[] times = Util.time(f, N, gen); - System.out.println(String.format("%35s, %50s, %10s", name, Util.stats(times), makeResString(times))); + String retS = makeResString(times); + System.out.println(String.format("%35s, %s, %10s", name, Util.stats(times), retS)); } protected abstract String makeResString(); diff --git a/src/test/java/org/apache/sysds/performance/compression/WriteTest.java b/src/test/java/org/apache/sysds/performance/compression/WriteTest.java index c81cae50009..567bab2817c 100644 --- a/src/test/java/org/apache/sysds/performance/compression/WriteTest.java +++ b/src/test/java/org/apache/sysds/performance/compression/WriteTest.java @@ -26,43 +26,53 @@ import org.apache.sysds.runtime.compress.colgroup.scheme.CompressionScheme; import org.apache.sysds.runtime.compress.lib.CLALibScheme; import org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; +import org.apache.sysds.runtime.matrix.data.LibMatrixMult; import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.test.TestUtils; public class WriteTest extends APerfTest { + final int k; + public WriteTest(int N, IGenerate gen) { super(N, gen); + k = 1; + } + + public WriteTest(int N, IGenerate gen, int k) { + super(N, gen); + this.k = k; } public void run() throws Exception, InterruptedException { System.out.println(this); - final int k = InfrastructureAnalyzer.getLocalParallelism(); - execute(() -> sumTask(1), "Warmup Sum task Single Thread"); - execute(() -> sumTask(k), "Warmup Sum task Parallel "); - execute(() -> compressTask(1), "Compression In Memory Single Thread"); - execute(() -> compressTask(k), "Compression In Memory Parallel "); - - // Not guaranteed to contain entire schema - final CompressionScheme sch = CLALibScheme.getScheme(getC()); - execute(() -> updateAndApplyScheme(sch, 1), "Update&Apply Scheme"); - execute(() -> updateAndApplySchemeFused(sch, 1), "Update&Apply Scheme Fused"); - execute(() -> applyScheme(sch, 1), "Apply Scheme"); - + warmup(() -> sumTask(k), N); + execute(() -> sumTask(k), "Sum"); + final MatrixBlock v = genVector(); + execute(() -> matrixVector(v, k), "MV mult"); + final CompressionScheme sch2 = CLALibScheme.getScheme(getC()); - execute(() -> updateAndApplyScheme(sch2, k), "Update&Apply Scheme Parallel"); - execute(() -> updateAndApplySchemeFused(sch2, k), "Update&Apply Scheme Fused Parallel"); - execute(() -> applyScheme(sch2, k), "Apply Scheme Parallel"); + execute(() -> updateAndApplyScheme(sch2, k), "Update&Apply Scheme"); + execute(() -> updateAndApplySchemeFused(sch2, k), "Update&Apply Scheme Fused"); + execute(() -> applyScheme(sch2, k), "Apply Scheme"); + execute(() -> fromEmptySchemeDoNotKeep(k), "Update&Apply from Empty"); + execute(() -> compressTask(k), "Normal Compression"); - execute(() -> fromEmptySchemeDoNotKeep(1), "From Empty"); - execute(() -> fromEmptySchemeDoNotKeep(k), "From Empty Parallel"); + } + private void matrixVector(MatrixBlock v, int k) { + MatrixBlock mb = gen.take(); + long in = mb.getInMemorySize(); + MatrixBlock r = LibMatrixMult.matrixMult(mb, v, k); + long out = r.getInMemorySize(); + ret.add(new InOut(in, out)); } private void sumTask(int k) { MatrixBlock mb = gen.take(); long in = mb.getInMemorySize(); - mb.sum(InfrastructureAnalyzer.getLocalParallelism()); - long out = 8; + MatrixBlock r = mb.sum(InfrastructureAnalyzer.getLocalParallelism()); + long out = r.getInMemorySize(); ret.add(new InOut(in, out)); } @@ -114,6 +124,13 @@ private CompressedMatrixBlock getC() throws InterruptedException { return (CompressedMatrixBlock) CompressedMatrixBlockFactory.compress(mb).getLeft(); } + private MatrixBlock genVector() throws InterruptedException { + gen.generate(1); + MatrixBlock mb = gen.take(); + MatrixBlock vector = TestUtils.generateTestMatrixBlock(mb.getNumColumns(), 1, -1.0, 1.0, 1.0, 324); + return vector; + } + @Override protected String makeResString() { throw new RuntimeException("Do not call"); @@ -124,43 +141,78 @@ protected String makeResString(double[] times) { double totalIn = 0; double totalOut = 0; double totalTime = 0.0; + for(int i = 0; i < ret.size(); i++) // set times + ret.get(i).time = times[i] / 1000; // ms to sec + + ret.sort(WriteTest::compare); - for(int i = 0; i < ret.size(); i++) { + final int l = ret.size(); + final int remove = (int)Math.floor((double)l * 0.05); + + final int el = l - remove *2; + + + for(int i = remove; i < ret.size() - remove; i++) { InOut e = ret.get(i); totalIn += e.in; totalOut += e.out; - totalTime += times[i]; + totalTime += e.time; } double bytePerMsIn = totalIn / totalTime; double bytePerMsOut = totalOut / totalTime; + // double meanTime = totalTime / el; - return String.format("%s In, %s Out", changeValue(bytePerMsIn), changeValue(bytePerMsOut)); - } + double varIn = 0; + double varOut = 0; + // double varTime = 0; - protected String changeValue(double bytePerMs) { - double bytePerSec = bytePerMs * 1000; - if(bytePerSec > 1000000000) { - return String.format("%6.2f GB/s", bytePerSec / 1024 / 1024 / 1024); - } - else if(bytePerSec > 1000000) { - return String.format("%6.2f MB/s", bytePerSec / 1024 / 1024); - } - else if(bytePerSec > 1000) { - return String.format("%6.2f KB/s", bytePerSec / 1024); - } - else { - return String.format("%6.2f B/s", bytePerSec); + for(int i = remove; i < ret.size() - remove; i++) { + InOut e = ret.get(i); + varIn += Math.pow(e.in / e.time - bytePerMsIn, 2); + varOut += Math.pow(e.out /e.time - bytePerMsOut, 2); } + + double stdIn = Math.sqrt(varIn / el); + double stdOut = Math.sqrt(varOut / el); + + return String.format("%12.0f+-%12.0f Byte/s, %12.0f+-%12.0f Byte/s", bytePerMsIn, stdIn, bytePerMsOut, stdOut); + } + + // protected String changeValue(double bytePerMs) { + // // double bytePerSec = bytePerMs * 1000; + // if(bytePerSec > 1000000000) { + // return String.format("%6.2f GB/s", bytePerSec / 1024 / 1024 / 1024); + // } + // else if(bytePerSec > 1000000) { + // return String.format("%6.2f MB/s", bytePerSec / 1024 / 1024); + // } + // else if(bytePerSec > 1000) { + // return String.format("%6.2f KB/s", bytePerSec / 1024); + // } + // else { + // return String.format("%6.2f B/s", bytePerSec); + // } + // } + + public static int compare(InOut a, InOut b) { + if(a.time == b.time) + return 0; + else if(a.time < b.time) + return -1; + else + return 1; } protected class InOut { protected long in; protected long out; + protected double time; protected InOut(long in, long out) { this.in = in; this.out = out; } + } }