phrasal_segmentation.sh

#!/usr/bin/env bash
SEGMENTATION_MODEL=results/segmentation.model
#TEXT_TO_SEG=${TEXT_TO_SEG:-data/EN/DBLP.5K.txt}
#TEXT_TO_SEG=${TEXT_TO_SEG:-data/20newsgroups.txt} #huangweijing
TEXT_TO_SEG=$1
HIGHLIGHT_MULTI=0.5
HIGHLIGHT_SINGLE=0.8
ENABLE_POS_TAGGING=1
THREAD=10

green=`tput setaf 2`
reset=`tput sgr0`

echo ${green}===Compilation===${reset}

#COMPILE=${COMPILE:- 1}
if [ ! -e bin/segphrase_segment ]; then
    bash compile.sh
fi

mkdir -p tmp
mkdir -p results

### END Compilation###

echo ${green}===Tokenization===${reset}

TOKENIZER="-cp .:tools/tokenizer/lib/*:tools/tokenizer/resources/:tools/tokenizer/build/ Tokenizer"
TOKENIZED_TEXT_TO_SEG=tmp/tokenized_text_to_seg.txt
CASE=tmp/case_tokenized_text_to_seg.txt
TOKEN_MAPPING=tmp/token_mapping.txt

echo -ne "Current step: Tokenizing input file...\033[0K\r"
time java $TOKENIZER -m direct_test -i $TEXT_TO_SEG -o $TOKENIZED_TEXT_TO_SEG -t $TOKEN_MAPPING -c N -thread $THREAD

LANGUAGE=`cat tmp/language.txt`
echo -ne "Detected Language: $LANGUAGE\033[0K\n"

### END Tokenization ###

echo ${green}===Part-Of-Speech Tagging===${reset}

if [ ! $LANGUAGE == "JA" ] && [ ! $LANGUAGE == "CN" ]  && [ ! $LANGUAGE == "OTHER" ]  && [ $ENABLE_POS_TAGGING -eq 1 ]; then
	RAW=tmp/raw_tokenized_text_to_seg.txt # TOKENIZED_TEXT_TO_SEG is the suffix name after "raw_"
	export THREAD LANGUAGE RAW
	bash ./tools/treetagger/pos_tag.sh
	mv tmp/pos_tags.txt tmp/pos_tags_tokenized_text_to_seg.txt
fi

POS_TAGS=tmp/pos_tags_tokenized_text_to_seg.txt

### END Part-Of-Speech Tagging ###

echo ${green}===Phrasal Segmentation===${reset}
echo "THREAD=$THREAD"
echo "SEGMENTATION_MODEL=$SEGMENTATION_MODEL"
echo "HIGHLIGHT_MULTI=$HIGHLIGHT_MULTI"
echo "HIGHLIGHT_SINGLE=$HIGHLIGHT_SINGLE"

if [ $ENABLE_POS_TAGGING -eq 1 ]; then
	time ./bin/segphrase_segment \
        --pos_tag \
        --thread $THREAD \
        --model $SEGMENTATION_MODEL \
		--highlight-multi $HIGHLIGHT_MULTI \
		--highlight-single $HIGHLIGHT_SINGLE \
        --filtered_phrases results/filtered_phrases.txt \
        --token_mapping tmp/token_mapping.txt
else
	time ./bin/segphrase_segment \
        --thread $THREAD \
        --model $SEGMENTATION_MODEL \
		--highlight-multi $HIGHLIGHT_MULTI \
		--highlight-single $HIGHLIGHT_SINGLE \
        --filtered_phrases results/filtered_phrases.txt \
        --token_mapping tmp/token_mapping.txt
fi

### END Segphrasing ###

echo ${green}===Generating Output===${reset}
java $TOKENIZER -m segmentation -i $TEXT_TO_SEG -segmented tmp/tokenized_segmented_sentences.txt -o results/segmentation.txt -tokenized_raw tmp/raw_tokenized_text_to_seg.txt -tokenized_id tmp/tokenized_text_to_seg.txt -c N

### END Generating Output for Checking Quality ###