-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
7 changed files
with
258 additions
and
109 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,7 @@ | ||
# Luigi client config | ||
|
||
[core] | ||
logging_conf_file: luigi_logging.ini | ||
|
||
[hadoop] | ||
command: hadoop | ||
# command: /root/ephemeral-hdfs/bin/hadoop |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
; Eggo "global" config | ||
|
||
[paths] | ||
; The location of the result data sets | ||
; supports S3 (s3n://), HDFS (hdfs://), and local (file://) targets | ||
; the specified URL will be the root of the eggo directory structure (see spec) | ||
eggo_base_url: s3n://bdg-eggo | ||
|
||
; Path to stage the raw input data on the target distributed fs | ||
raw_data_prefix: raw | ||
|
||
; Path to store tmp/intermediate data in the target distributed fs | ||
; tmp data always ends up in eggo_base_url/tmp/dataset_name/tmp_data_prefix/... | ||
;tmp_data_prefix: ; if not set manually, module load generates random prefix | ||
|
||
; Absolute path to store tmp data on remote worker machines' local file systems | ||
; e.g., on EC2 machines, this could be the ephemeral drive mount point /mnt | ||
worker_local_tmp_data_dir: /mnt | ||
|
||
|
||
[hadoop] | ||
; NOTE: make sure the Luigi hadoop config is appropriately set in client.cfg | ||
hadoop_home: /root/ephemeral-hdfs ; overridden by HADOOP_HOME env var | ||
streaming_jar: ; this will override the similar option in Luigi's client.cfg | ||
|
||
[aws] | ||
; These will all be overridden by corresponding env variables (in ALL_CAPS) | ||
aws_access_key_id: <MY_ACCESS_KEY> | ||
aws_secret_access_key: <MY_SECRET_KEY> | ||
ec2_key_pair: <MY_KEY> | ||
ec2_private_key_file: <MY_KEY_FILE> | ||
|
||
|
||
[director] | ||
; TODO: add docs to these options | ||
region: us-east-1 | ||
launcher_instance_type: m3.large | ||
launcher_ami: ami-a25415cb ; RHEL 6.4 x86 | ||
cluster_ami: %(launcher_ami)s | ||
stack_name: bdg-eggo-%(ec2_key_pair)s | ||
; the pointers to the director configs are executed relative to CWD (which may | ||
; be the same as EGGO_HOME); set them to an absolute path if desired, or use | ||
; "%(eggo_home)s" to access the EGGO_HOME env variable | ||
cloudformation_template: conf/director/cfn-cloudera-us-east-1-public-subnet.template | ||
director_conf_template: conf/director/aws.conf |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.