Skip to content

Commit

Permalink
compare
Browse files Browse the repository at this point in the history
  • Loading branch information
@Baunsgaard
Baunsgaard committed Aug 18, 2023
1 parent 3d0f14c commit 60f6282
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 199 deletions.
9 changes: 8 additions & 1 deletion src/main/java/org/apache/sysds/runtime/compress/utils/DblArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,14 @@ public void resetHash() {
}

public boolean isEmpty() {
return _arr == null;
if(_arr == null)
return true;

for(int i = 0; i < _arr.length; i++) {
if(_arr[i] != 0)
return false;
}
return true;
}

@Override
Expand Down
185 changes: 0 additions & 185 deletions src/test/java/org/apache/sysds/test/component/compress/readers/ReadersTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@
package org.apache.sysds.test.component.compress.readers;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.fail;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Expand All @@ -32,7 +30,6 @@
import org.apache.sysds.runtime.compress.utils.DblArrayCountHashMap;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.test.TestUtils;
import org.junit.Ignore;
import org.junit.Test;

public class ReadersTest {
Expand Down Expand Up @@ -136,186 +133,4 @@ public void testInvalidRange_02() {
mb.allocateDenseBlock();
ReaderColumnSelection.createReader(mb, ColIndexFactory.create(2), false, 10, 9);
}

@Test
public void isEmptyNan() {
try {

MatrixBlock mb = new MatrixBlock(10, 5, Double.NaN);

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mb, ColIndexFactory.create(2), false, 0,
mb.getNumRows());
assertEquals(null, reader.nextRow());
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}

@Test
public void isNaN() {
try {

MatrixBlock mb = new MatrixBlock(10, 5, Double.NaN);
mb.setValue(1, 1, 3214);

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mb, ColIndexFactory.create(2), false, 0,
mb.getNumRows());
DblArray a = reader.nextRow();
assertNotEquals(null, a);
assertEquals(3214.0, a.getData()[1], 0.0);
assertEquals(0.0, a.getData()[0], 0.0);
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}

@Test
public void isEmptyNanTransposed() {
try {

MatrixBlock mb = new MatrixBlock(10, 5, Double.NaN);

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mb, ColIndexFactory.create(2), true, 0,
mb.getNumRows());
assertEquals(null, reader.nextRow());
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}

@Test
public void isNaNTransposed() {
try {

MatrixBlock mb = new MatrixBlock(10, 5, Double.NaN);
mb.setValue(1, 1, 3214);

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mb, ColIndexFactory.create(2), true, 0,
mb.getNumRows());
DblArray a = reader.nextRow();
assertNotEquals(null, a);
assertEquals(3214.0, a.getData()[1], 0.0);
assertEquals(0.0, a.getData()[0], 0.0);
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}

@Test
public void isEmptyNanMultiBlock() {
try {

MatrixBlock mb = ReadersTestCompareReaders.createMock(new MatrixBlock(10, 5, Double.NaN));

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mb, ColIndexFactory.create(2), false, 0,
mb.getNumRows());
assertEquals(null, reader.nextRow());
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}

@Test
public void isNaNMultiBlock() {
try {

MatrixBlock mb = ReadersTestCompareReaders.createMock(new MatrixBlock(10, 5, Double.NaN));
mb.setValue(1, 1, 3214);

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mb, ColIndexFactory.create(2), false, 0,
mb.getNumRows());
DblArray a = reader.nextRow();
assertNotEquals(null, a);
assertEquals(3214.0, a.getData()[1], 0.0);
assertEquals(0.0, a.getData()[0], 0.0);
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}

@Test
public void isEmptyNanMultiBlockTransposed() {
try {

MatrixBlock mb = ReadersTestCompareReaders.createMock(new MatrixBlock(10, 5, Double.NaN));

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mb, ColIndexFactory.create(2), true, 0,
mb.getNumRows());
assertEquals(null, reader.nextRow());
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}

@Test
public void isNaNMultiBlockTransposed() {
try {

MatrixBlock mb = ReadersTestCompareReaders.createMock(new MatrixBlock(10, 5, Double.NaN));
mb.setValue(1, 1, 3214);

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mb, ColIndexFactory.create(2), true, 0,
mb.getNumRows());
DblArray a = reader.nextRow();
assertNotEquals(null, a);
assertEquals(3214.0, a.getData()[1], 0.0);
assertEquals(0.0, a.getData()[0], 0.0);
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}

@Test
public void isNanSparseBlock() {
MatrixBlock mbs = new MatrixBlock(10, 10, true);
mbs.setValue(1, 1, 3214);
mbs.setValue(0, 0, Double.NaN);
mbs.setValue(0, 1, Double.NaN);
mbs.setValue(1, 0, Double.NaN);

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mbs, ColIndexFactory.create(2), false, 0,
mbs.getNumRows());

DblArray a = reader.nextRow();
assertNotEquals(null, a);
assertEquals(3214.0, a.getData()[1], 0.0);
assertEquals(0.0, a.getData()[0], 0.0);
assertEquals(null, reader.nextRow());
}

@Test
// for now ignore.. i need a better way of reading matrices containing Nan Becuase the check is very expensive
@Ignore
public void isNanSparseBlockTransposed() {
MatrixBlock mbs = new MatrixBlock(10, 10, true);
mbs.setValue(1, 1, 3214);
mbs.setValue(0, 0, Double.NaN);
mbs.setValue(0, 1, Double.NaN);
mbs.setValue(1, 0, Double.NaN);

ReaderColumnSelection reader = ReaderColumnSelection.createReader(mbs, ColIndexFactory.create(2), true, 0,
mbs.getNumRows());

DblArray a = reader.nextRow();
assertNotEquals(null, a);
assertEquals(3214.0, a.getData()[1], 0.0);
assertEquals(0.0, a.getData()[0], 0.0);
assertEquals(null, reader.nextRow());
}
}
74 changes: 61 additions & 13 deletions ...test/java/org/apache/sysds/test/component/compress/readers/ReadersTestCompareReaders.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.sysds.test.component.compress.readers;

import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.util.ArrayList;
Expand Down Expand Up @@ -73,7 +74,8 @@ public static Collection<Object[]> data() {
for(int c = 0; c < cols.length; c++) {
for(int r = 0; r < rows.length; r++) {
mb = TestUtils.generateTestMatrixBlock(rows[r], cols[c], 1, 10, spar[s], i);
tests.add(new Object[] {mb});
if(!mb.isInSparseFormat())
tests.add(new Object[] {mb});
}
}
}
Expand All @@ -83,13 +85,15 @@ public static Collection<Object[]> data() {
mb = new MatrixBlock(100, 10, false);
for(int i = 0; i < 10; i++)
mb.quickSetValue(3, i, 231);
tests.add(new Object[] {mb});
if(!mb.isInSparseFormat())
tests.add(new Object[] {mb});

// only one col is set
mb = new MatrixBlock(100, 10, false);
for(int i = 0; i < 100; i++)
mb.quickSetValue(i, 4, 231);
tests.add(new Object[] {mb});
if(!mb.isInSparseFormat())
tests.add(new Object[] {mb});

return tests;
}
Expand Down Expand Up @@ -498,7 +502,8 @@ public void testCompareDenseTransposedLargeFewRowsFromEnd() {
final int end = m.getNumRows() - 10;
final int start = end - 10;
ReaderColumnSelection a = ReaderColumnSelection.createReader(m, cols, false, start, end);
ReaderColumnSelection b = ReaderColumnSelection.createReader(mMockLargeTransposed, cols, true, start, end);
ReaderColumnSelection b = ReaderColumnSelection.createReader(mMockLargeTransposed, cols, true, start,
end);
compareReaders(a, b, start, end);
}
}
Expand Down Expand Up @@ -527,12 +532,32 @@ private void compareReaders(ReaderColumnSelection a, ReaderColumnSelection b) {
DblArray br = null;
while((ar = a.nextRow()) != null) {
br = b.nextRow();
final int aIdx = a.getCurrentRowIndex();
final int bIdx = b.getCurrentRowIndex();
int aIdx = a.getCurrentRowIndex();
int bIdx = b.getCurrentRowIndex();
while(aIdx < bIdx && ar != null) {
assertTrue(ar == null || ar.isEmpty());
ar = a.nextRow();
aIdx = a.getCurrentRowIndex();
}
while(bIdx < aIdx && br != null) {
assertTrue(br == null || br.isEmpty());
br = b.nextRow();
bIdx = b.getCurrentRowIndex();
}
if(aIdx != bIdx)
fail("Not equal row indexes" + aIdx + " " + bIdx);
if(!ar.equals(br))
fail("Not equal row values" + ar + " " + br + " at row: " + aIdx);
fail("Not equal row indexes: " + aIdx + " " + bIdx);

if(ar == null && br == null) {
// all good
}
else if(ar == null && br != null && br.isEmpty()) {
// all good
}
else if(br == null && ar != null && ar.isEmpty()) {
// all good
}
else if(ar != null && br != null && !ar.equals(br))
fail("Not equal row values: " + ar + " " + br + " at row: " + aIdx);
}
br = b.nextRow();

Expand All @@ -559,8 +584,19 @@ private void compareReaders(final ReaderColumnSelection a, final ReaderColumnSel
DblArray br = null;
while((ar = a.nextRow()) != null) {
br = b.nextRow();
final int aIdx = a.getCurrentRowIndex();
final int bIdx = b.getCurrentRowIndex();
int aIdx = a.getCurrentRowIndex();
int bIdx = b.getCurrentRowIndex();
while(aIdx < bIdx && ar != null) {
assertTrue(ar == null || ar.isEmpty());
ar = a.nextRow();
aIdx = a.getCurrentRowIndex();
}

while(bIdx < aIdx && br != null) {
assertTrue(br == null || br.isEmpty());
br = b.nextRow();
bIdx = b.getCurrentRowIndex();
}

if(aIdx != bIdx)
fail("Not equal row indexes" + aIdx + " " + bIdx);
Expand All @@ -569,8 +605,19 @@ private void compareReaders(final ReaderColumnSelection a, final ReaderColumnSel
if(aIdx >= end)
fail("reader violated the row upper " + aIdx + " is larger or equal to " + end + " "
+ b.getClass().getSimpleName());
if(!ar.equals(br))

if(ar == null && br == null) {
// all good
}
else if(ar == null && br != null && br.isEmpty()) {
// all good
}
else if(br == null && ar != null && ar.isEmpty()) {
// all good
}
else if(ar != null && br != null && !ar.equals(br)) {
fail("Not equal row values" + ar + " " + br + " at row: " + aIdx);
}
}
br = b.nextRow();

Expand Down Expand Up @@ -600,7 +647,8 @@ private IColIndex createColIdx(int start, int end, int skip) {
}

public static MatrixBlock createMock(MatrixBlock d) {
DenseBlockFP64 a = new DenseBlockFP64Mock(new int[] {d.getNumRows(), d.getNumColumns()}, d.getDenseBlockValues());
DenseBlockFP64 a = new DenseBlockFP64Mock(new int[] {d.getNumRows(), d.getNumColumns()},
d.getDenseBlockValues());
MatrixBlock b = new MatrixBlock(d.getNumRows(), d.getNumColumns(), a);
b.setNonZeros(d.getNumRows() * d.getNumColumns());
return b;
Expand Down

0 comments on commit 60f6282

Please sign in to comment.