-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #53 from laserson/EGGO-18-config
[EGGO-18] WIP: Setup config files
- Loading branch information
Showing
17 changed files
with
673 additions
and
474 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,92 @@ | ||
; Eggo "global" config | ||
|
||
; NOTE: to access the EGGO_HOME env variable, use "%(eggo_home)s" | ||
|
||
[core] | ||
; The location of the result data sets | ||
; supports S3 (s3n://), HDFS (hdfs://), and local (file://) targets | ||
; the specified URL will be the root of the eggo directory structure (see spec) | ||
eggo_base_url: s3n://bdg-eggo | ||
|
||
; The context for execution | ||
; Possible values are spark_ec2, director, local | ||
execution: spark_ec2 | ||
|
||
[versions] | ||
eggo_fork: laserson | ||
eggo_branch: EGGO-18-config | ||
adam_fork: bigdatagenomics | ||
adam_branch: master | ||
maven: 3.2.5 | ||
|
||
|
||
[paths] | ||
; Path to stage the raw input data on the target distributed fs | ||
; raw data ends up in <eggo_base_url>/<dfs_raw_data_prefix>/<dataset_name> | ||
dfs_raw_data_prefix: raw | ||
|
||
; Path to store tmp/intermediate data in the target distributed fs | ||
; tmp data ends up in <eggo_base_url>/<dfs_tmp_data_dir>/<dataset_name>/<random_id> | ||
dfs_tmp_data_prefix: tmp | ||
|
||
; Absolute path to a location on the remote worker's local fs where eggo can | ||
; write data to. e.g., on EC2 machines, this could be the ephemeral drive mount | ||
; point /mnt | ||
worker_data_dir: /mnt | ||
|
||
; Random identifier that is generated on module load. Do not set this manually | ||
;random_id: <generated-on-load> | ||
|
||
|
||
[local_env] | ||
; Can be overridden by setting SPARK_HOME env var | ||
spark_home: $SPARK_HOME ; hack: this var gets interp into shell cmds so env var will fill it | ||
|
||
|
||
[worker_env] | ||
eggo_config_worker_path: /mnt/eggo.cfg ; current file is copied to remote location | ||
|
||
; Comma-separated list of files to source in .bash_profile on remote worker | ||
files_to_source: /root/spark-ec2/ec2-variables.sh | ||
|
||
; Environment variables to set on the worker nodes in .bash_profiletheir values | ||
; can depend on variables set in %(files_to_source)s, as they will be sourced | ||
; first | ||
hadoop_home: /root/ephemeral-hdfs | ||
spark_home: /root/spark | ||
streaming_jar: %(hadoop_home)s/contrib/streaming/hadoop-streaming-1.0.4.jar | ||
spark_master_uri: spark://$MASTERS:7077 | ||
;eggo_home: <not configurable, but set as EGGO_HOME> | ||
;luigi_config_path: <not configurable, but set as $EGGO_HOME/conf/luigi/luigi.cfg | ||
|
||
|
||
[aws] | ||
; These can be set/overridden by setting corresponding local env vars (in ALL_CAPS) | ||
;aws_access_key_id: <MY_ACCESS_KEY> | ||
;aws_secret_access_key: <MY_SECRET_KEY> | ||
;ec2_key_pair: <MY_KEY> | ||
;ec2_private_key_file: <MY_KEY_FILE> | ||
|
||
|
||
[spark_ec2] | ||
; Spark EC2 scripts configuration | ||
; Path to local Spark installation | ||
region: us-east-1 | ||
instance_type: r3.2xlarge | ||
num_slaves: 2 | ||
user: root | ||
|
||
|
||
[director] | ||
; Cloudera Director configuration | ||
; TODO: add docs to these options | ||
region: us-east-1 | ||
launcher_instance_type: m3.large | ||
launcher_ami: ami-a25415cb ; RHEL 6.4 x86 | ||
cluster_ami: %(launcher_ami)s | ||
stack_name: bdg-eggo | ||
; the pointers to the director configs are executed relative to CWD (which may | ||
; be the same as EGGO_HOME); set them to an absolute path if desired, or use | ||
; "%(eggo_home)s" to access the EGGO_HOME env variable | ||
cloudformation_template: conf/director/cfn-cloudera-us-east-1-public-subnet.template | ||
director_conf_template: conf/director/aws.conf |
File renamed without changes.
Oops, something went wrong.