-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathsetup_data.sh
More file actions
executable file
·16 lines (16 loc) · 1008 Bytes
/
Copy pathsetup_data.sh
File metadata and controls
executable file
·16 lines (16 loc) · 1008 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#!/bin/bash
bunzip2 -k normalized_jokes.json.bz2
mkdir -p data
mv normalized_jokes.json data
python make_training_data.py
[[ -d subword-nmt ]] || git clone git@github.com:rsennrich/subword-nmt.git
cd subword-nmt
cat ../data/joke_setups.trn ../data/joke_punchlines.trn ../data/oneliners.trn | ./subword_nmt/learn_bpe.py -s 10000 > ../data/codes.bpe
./subword_nmt/apply_bpe.py -c ../data/codes.bpe < ../data/joke_setups.trn > ../data/joke_setups.trn.bpe
./subword_nmt/apply_bpe.py -c ../data/codes.bpe < ../data/joke_punchlines.trn > ../data/joke_punchlines.trn.bpe
./subword_nmt/apply_bpe.py -c ../data/codes.bpe < ../data/joke_setups.tst > ../data/joke_setups.tst.bpe
./subword_nmt/apply_bpe.py -c ../data/codes.bpe < ../data/joke_punchlines.tst > ../data/joke_punchlines.tst.bpe
./subword_nmt/apply_bpe.py -c ../data/codes.bpe < ../data/oneliners.trn > ../data/oneliners.trn.bpe
./subword_nmt/apply_bpe.py -c ../data/codes.bpe < ../data/oneliners.tst > ../data/oneliners.tst.bpe
cd ..
python make_vocabs.py