Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@Baunsgaard
Baunsgaard committed Sep 5, 2023
1 parent 0a6e476 commit cb70df6
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 54 deletions.
12 changes: 8 additions & 4 deletions src/main/java/org/apache/sysds/lops/Sql.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,17 @@

package org.apache.sysds.lops;

import org.apache.sysds.parser.DataExpression;
import org.apache.sysds.runtime.instructions.InstructionUtils;
import static org.apache.sysds.parser.DataExpression.SQL_CONN;
import static org.apache.sysds.parser.DataExpression.SQL_PASS;
import static org.apache.sysds.parser.DataExpression.SQL_QUERY;
import static org.apache.sysds.parser.DataExpression.SQL_USER;

import java.util.HashMap;

import static org.apache.sysds.common.Types.*;
import static org.apache.sysds.parser.DataExpression.*;
import org.apache.sysds.common.Types.DataType;
import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.parser.DataExpression;
import org.apache.sysds.runtime.instructions.InstructionUtils;

public class Sql extends Lop {
private HashMap<String, Lop> _inputParams;
Expand Down
43 changes: 29 additions & 14 deletions ...main/java/org/apache/sysds/runtime/controlprogram/paramserv/FederatedPSControlThread.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,19 @@

package org.apache.sysds.runtime.controlprogram.paramserv;

import static org.apache.sysds.runtime.util.ProgramConverter.NEWLINE;
import static org.apache.sysds.runtime.util.ProgramConverter.PROG_BEGIN;
import static org.apache.sysds.runtime.util.ProgramConverter.PROG_END;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
Expand All @@ -34,32 +47,34 @@
import org.apache.sysds.runtime.controlprogram.ProgramBlock;
import org.apache.sysds.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysds.runtime.controlprogram.federated.*;
import org.apache.sysds.runtime.controlprogram.federated.FederatedData;
import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest;
import org.apache.sysds.runtime.controlprogram.federated.FederatedRequest.RequestType;
import org.apache.sysds.runtime.controlprogram.federated.FederatedResponse;
import org.apache.sysds.runtime.controlprogram.federated.FederatedUDF;
import org.apache.sysds.runtime.controlprogram.federated.FederationUtils;
import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.PublicKey;
import org.apache.sysds.runtime.controlprogram.paramserv.homomorphicEncryption.SEALClient;
import org.apache.sysds.runtime.controlprogram.parfor.stat.Timing;
import org.apache.sysds.runtime.functionobjects.Multiply;
import org.apache.sysds.runtime.instructions.Instruction;
import org.apache.sysds.runtime.instructions.InstructionUtils;
import org.apache.sysds.runtime.instructions.cp.*;
import org.apache.sysds.runtime.instructions.cp.BooleanObject;
import org.apache.sysds.runtime.instructions.cp.CPOperand;
import org.apache.sysds.runtime.instructions.cp.CiphertextMatrix;
import org.apache.sysds.runtime.instructions.cp.Data;
import org.apache.sysds.runtime.instructions.cp.DoubleObject;
import org.apache.sysds.runtime.instructions.cp.FunctionCallCPInstruction;
import org.apache.sysds.runtime.instructions.cp.IntObject;
import org.apache.sysds.runtime.instructions.cp.ListObject;
import org.apache.sysds.runtime.instructions.cp.PlaintextMatrix;
import org.apache.sysds.runtime.instructions.cp.StringObject;
import org.apache.sysds.runtime.lineage.LineageItem;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.operators.RightScalarOperator;
import org.apache.sysds.runtime.lineage.LineageItem;
import org.apache.sysds.runtime.util.ProgramConverter;
import org.apache.sysds.utils.stats.ParamServStatistics;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.apache.sysds.runtime.util.ProgramConverter.*;

public class FederatedPSControlThread extends PSWorker implements Callable<Void> {
private static final long serialVersionUID = 6846648059569648791L;
protected static final Log LOG = LogFactory.getLog(ParamServer.class.getName());
Expand Down
22 changes: 18 additions & 4 deletions src/main/java/org/apache/sysds/runtime/util/AutoDiff.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,19 @@

package org.apache.sysds.runtime.util;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Stack;

import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.sysds.common.Types;
import org.apache.sysds.hops.*;
import org.apache.sysds.hops.DataGenOp;
import org.apache.sysds.hops.DataOp;
import org.apache.sysds.hops.Hop;
import org.apache.sysds.hops.LiteralOp;
import org.apache.sysds.hops.ReorgOp;
import org.apache.sysds.hops.recompile.Recompiler;
import org.apache.sysds.hops.rewrite.HopRewriteUtils;
import org.apache.sysds.parser.DataExpression;
Expand All @@ -34,14 +44,18 @@
import org.apache.sysds.runtime.instructions.Instruction;
import org.apache.sysds.runtime.instructions.InstructionParser;
import org.apache.sysds.runtime.instructions.InstructionUtils;
import org.apache.sysds.runtime.instructions.cp.*;
import org.apache.sysds.runtime.instructions.cp.CPInstruction;
import org.apache.sysds.runtime.instructions.cp.CPOperand;
import org.apache.sysds.runtime.instructions.cp.Data;
import org.apache.sysds.runtime.instructions.cp.DataGenCPInstruction;
import org.apache.sysds.runtime.instructions.cp.ListObject;
import org.apache.sysds.runtime.instructions.cp.ScalarObjectFactory;
import org.apache.sysds.runtime.instructions.cp.VariableCPInstruction;
import org.apache.sysds.runtime.instructions.spark.RandSPInstruction;
import org.apache.sysds.runtime.lineage.LineageItem;
import org.apache.sysds.runtime.lineage.LineageParser;
import org.apache.sysds.utils.Explain;

import java.util.*;

public class AutoDiff {
private static final String ADVARPREFIX = "adVar";
private static final boolean DEBUG = false;
Expand Down
32 changes: 0 additions & 32 deletions src/main/java/org/apache/sysds/runtime/util/DataConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,6 @@
* This class provides methods to read and write matrix blocks from to HDFS using different data formats.
* Those functionalities are used especially for CP read/write and exporting in-memory matrices to HDFS
* (before executing MR jobs).
*
*/
public class DataConverter {
private static final String DELIM = " ";
Expand Down Expand Up @@ -112,20 +111,6 @@ public static void writeTensorToHDFS(TensorBlock tensor, String dir, FileFormat
writer.writeTensorToHDFS(tensor, dir, blen);
}

// public static MatrixBlock readMatrixFromHDFS(String dir, FileFormat fmt, long rlen, long clen, int blen, boolean localFS)
// throws IOException
// {
// ReadProperties prop = new ReadProperties();

// prop.path = dir;
// prop.fmt = fmt;
// prop.rlen = rlen;
// prop.clen = clen;
// prop.blen = blen;
// prop.localFS = localFS;

// return readMatrixFromHDFS(prop);
// }

public static MatrixBlock readMatrixFromHDFS(String dir, FileFormat fmt, long rlen, long clen, int blen)
throws IOException
Expand Down Expand Up @@ -156,23 +141,6 @@ public static MatrixBlock readMatrixFromHDFS(String dir, FileFormat fmt, long rl
return readMatrixFromHDFS(prop);
}

// public static MatrixBlock readMatrixFromHDFS(String dir, FileFormat fmt, long rlen, long clen,
// int blen, long expectedNnz, boolean localFS)
// throws IOException
// {
// ReadProperties prop = new ReadProperties();

// prop.path = dir;
// prop.fmt = fmt;
// prop.rlen = rlen;
// prop.clen = clen;
// prop.blen = blen;
// prop.expectedNnz = expectedNnz;
// prop.localFS = localFS;

// return readMatrixFromHDFS(prop);
// }

public static MatrixBlock readMatrixFromHDFS(String dir, FileFormat fmt, long rlen, long clen,
int blen, long expectedNnz, FileFormatProperties formatProperties)
throws IOException
Expand Down

0 comments on commit cb70df6

Please sign in to comment.