Skip to content

Commit

Permalink
a bit more sleep to give IO a chance in the masively overusing compr…
Browse files Browse the repository at this point in the history
…ession tests
  • Loading branch information
Baunsgaard committed Aug 20, 2023
1 parent 169c612 commit a94ef35
Showing 1 changed file with 4 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ private void testWriteSparkReadCP(MatrixBlock mb, int blen1, int blen2) {
Timing t = new Timing();
String f1 = getName();
WriterCompressed.writeCompressedMatrixToHDFS(mb, f1, blen1);

Thread.sleep(100);
// Make sure the first file is written
File f = new File(f1);
assertTrue(f.isFile() || f.isDirectory());
Expand All @@ -293,6 +293,7 @@ private void testWriteSparkReadCP(MatrixBlock mb, int blen1, int blen2) {
}
else
WriterCompressed.writeRDDToHDFS(m, f2);
Thread.sleep(100);

// Read locally the spark block written.
MatrixBlock mbr = IOCompressionTestUtils.read(f2);
Expand All @@ -312,7 +313,7 @@ private void testReblock(MatrixBlock mb, int blen1, int blen2) {

String f1 = getName();
WriterCompressed.writeCompressedMatrixToHDFS(mb, f1, blen1);

Thread.sleep(100);
// Read in again as RDD
JavaPairRDD<MatrixIndexes, MatrixBlock> m = getRDD(f1); // Our starting point

Expand Down Expand Up @@ -361,6 +362,7 @@ public void readRDDThroughSparkExecutionContext(MatrixBlock mb, int blen) {
String n = getName();

WriterCompressed.writeCompressedMatrixToHDFS(mb, n, blen);
Thread.sleep(100);
MatrixReader r = ReaderCompressed.create();
MatrixBlock mb2 = r.readMatrixFromHDFS(n, (long) mb.getNumRows(), (long) mb.getNumColumns(), blen, -1L);
TestUtils.compareMatricesBitAvgDistance(mb, mb2, 0, 0);
Expand Down

0 comments on commit a94ef35

Please sign in to comment.