ERROR while pulling python execute.py

.gitignore

@@ -1,2 +1,5 @@
 data/
+working_dir/
 *.pyc
+.venv/
+.idea/

README.md

@@ -20,22 +20,81 @@ Use [pip](https://pypi.python.org/pypi/pip) to install any missing dependencies
 Usage
 ===========
 
-To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
+Create venv & install dependencies:
+
+> Using viritualenv here to be compatable with python2, install with "pip install virtualenv"
+
+```
+# create venv
+python -m virtualenv .venv
+
+# enter venv (assuming macos/linux)
+source .venv/bin/activate
+
+# install requirements
+pip install -r requirements.txt
+```
+
+Training
+--------------------
+
+0. Create directories
+ ```
+ mkdir working_dir
+ mkdir data
+ ```
+
+1. Download the [Cornell Movie Dialogue dataset](https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html) and place the unzipped content in the `data/` directory.
+ ```
+ cd data
+ wget http://www.mpi-sws.org/~cristian/data/cornell_movie_dialogs_corpus.zip
+ unzip cornell_movie_dialogs_corpus.zip
+
+ # move to `data/` root
+ mv "corenell movie-dialogs corpus"/* .
+ ```
+
+2. Prepare data for training, in the `data/` directory, run the `prepare_data.py` script
+ ```
+ # move to project root
+ cd ..
+ python prepare_data.py
+ ```
+
+3. To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
 
 `mode = train`
 
-then run the code like so
+4. Start training, by running the code like so:
+
+ ``python execute.py``
+
+> There is no mechanism to stop training, you will need to 'ctrl-c' to stop training after a period of time.
+
+
+Test
+-------------
+
+1. To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+
+ `mode = test`
 
-``python execute.py``
+2. To test run the code like so:
 
-To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+ ```
+ python execute.py
 
-`mode = test`
+ >> Mode : test
 
-then run the code like so
+ Reading model parameters from working_dir/seq2seq.ckpt-10200
+ > 
+ ```
 
-``python execute.py``
+
 
+3. Confrim... 
+ - What does "Test do?"
+ - How to use it?
 
 Challenge
 ===========

data_utils.py

@@ -20,16 +20,16 @@
 
 import os
 import re
-
-from six.moves import urllib
+from io import open
+from collections import Counter
 
 from tensorflow.python.platform import gfile
 
 # Special vocabulary symbols - we always put them at the start.
-_PAD = b"_PAD"
-_GO = b"_GO"
-_EOS = b"_EOS"
-_UNK = b"_UNK"
+_PAD = "_PAD"
+_GO = "_GO"
+_EOS = "_EOS"
+_UNK = "_UNK"
 _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
 
 PAD_ID = 0
@@ -38,93 +38,97 @@
 UNK_ID = 3
 
 # Regular expressions used to tokenize.
-_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
-_DIGIT_RE = re.compile(br"\d")
+_WORD_SPLIT = re.compile("([.,!?\"':;)(])")
+_DIGIT_RE = re.compile(r"\d")
 
+CORNELL_MOVIE_CORPUS_ENCODING = 'ISO-8859-2'
 
 
 def basic_tokenizer(sentence):
- """Very basic tokenizer: split the sentence into a list of tokens."""
- words = []
- for space_separated_fragment in sentence.strip().split():
- words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
- return [w for w in words if w]
+ """Very basic tokenizer: split the sentence into a list of tokens."""
+ all_words = []
+ for space_separated_fragment in sentence.strip().split():
+ words = re.split(_WORD_SPLIT, space_separated_fragment)
+ for word in words:
+ if word:
+ all_words.append(word)
+ return all_words
 
 
 def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(vocabulary_path):
- print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ if not tokenizer:
+ tokenizer = basic_tokenizer
+
+ if not os.path.exists(vocabulary_path):
+ print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ vocab = Counter()
+ with open(data_path, 'rt', encoding='utf8') as f:
+ for counter, sentence in enumerate(f, 1):
+ if counter % 100000 == 0:
+ print(" processing line %d" % counter)
+ tokens = tokenizer(sentence)
+ for w in tokens:
+ if normalize_digits:
+ word = re.sub(_DIGIT_RE, '0', w)
+ else:
+ word = w
+ vocab[word] += 1
+
+ vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
+ print('>> Full Vocabulary Size :', len(vocab_list))
+ if len(vocab_list) > max_vocabulary_size:
+ vocab_list = vocab_list[:max_vocabulary_size]
+ print('>>>> Vocab Truncated to: {}'.format(max_vocabulary_size))
+ with open(vocabulary_path, 'wt', encoding='utf8') as vocab_file:
+ for w in vocab_list:
+ vocab_file.write(w + '\n')
+
+
+def initialize_vocabulary(vocabulary_path, encoding=CORNELL_MOVIE_CORPUS_ENCODING):
  vocab = {}
- with gfile.GFile(data_path, mode="rb") as f:
- counter = 0
- for line in f:
- counter += 1
- if counter % 100000 == 0:
- print(" processing line %d" % counter)
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1
- else:
- vocab[word] = 1
- vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
- print('>> Full Vocabulary Size :',len(vocab_list))
- if len(vocab_list) > max_vocabulary_size:
- vocab_list = vocab_list[:max_vocabulary_size]
- with gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
- for w in vocab_list:
- vocab_file.write(w + b"\n")
-
-
-def initialize_vocabulary(vocabulary_path):
-
- if gfile.Exists(vocabulary_path):
  rev_vocab = []
- with gfile.GFile(vocabulary_path, mode="rb") as f:
- rev_vocab.extend(f.readlines())
- rev_vocab = [line.strip() for line in rev_vocab]
- vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
+ if gfile.Exists(vocabulary_path):
+ with open(vocabulary_path, 'rt', encoding=encoding) as f:
+ for index, line in enumerate(f, 1):
+ element = line.strip()
+ rev_vocab.append(element)
+ vocab[element] = index
+ assert len(vocab) == len(rev_vocab)
+ if not (vocab and rev_vocab):
+ raise ValueError('File empty: {}'.format(vocabulary_path))
+ else:
+ raise ValueError("Vocabulary file %s not found.", vocabulary_path)
  return vocab, rev_vocab
- else:
- raise ValueError("Vocabulary file %s not found.", vocabulary_path)
 
 
 def sentence_to_token_ids(sentence, vocabulary, tokenizer=None, normalize_digits=True):
-
-if tokenizer:
+ if not tokenizer:
+tokenizer = basic_tokenizer
  words = tokenizer(sentence)
- else:
- words = basic_tokenizer(sentence)
- if not normalize_digits:
- return [vocabulary.get(w, UNK_ID) for w in words]
- # Normalize digits by 0 before looking words up in the vocabulary.
- return [vocabulary.get(re.sub(_DIGIT_RE, b"0", w), UNK_ID) for w in words]
+ if not normalize_digits:
+ return [vocabulary.get(w, UNK_ID) for w in words]
+ # Normalize digits by 0 before looking words up in the vocabulary.
+ return [vocabulary.get(re.sub(_DIGIT_RE, '0', w), UNK_ID) for w in words]
 
 
 def data_to_token_ids(data_path, target_path, vocabulary_path,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(target_path):
- print("Tokenizing data in %s" % data_path)
- vocab, _ = initialize_vocabulary(vocabulary_path)
- with gfile.GFile(data_path, mode="rb") as data_file:
- with gfile.GFile(target_path, mode="w") as tokens_file:
- counter = 0
- for line in data_file:
- counter += 1
- if counter % 100000 == 0:
- print(" tokenizing line %d" % counter)
- token_ids = sentence_to_token_ids(line, vocab, tokenizer,
- normalize_digits)
- tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
-
-
-
-def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size, dec_vocabulary_size, tokenizer=None):
-
+ if not gfile.Exists(target_path):
+ print("Tokenizing data in %s" % data_path)
+ vocab, _ = initialize_vocabulary(vocabulary_path)
+ with gfile.GFile(data_path, mode="rb") as data_file:
+ with gfile.GFile(target_path, mode="w") as tokens_file:
+ for counter, line in enumerate(data_file, 1):
+ if counter % 100000 == 0:
+ print(" tokenizing line %d" % counter)
+ token_ids = sentence_to_token_ids(line, vocab, tokenizer,
+ normalize_digits)
+ tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
+
+
+def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size,
+ dec_vocabulary_size, tokenizer=None):
  # Create vocabularies of the appropriate sizes.
  enc_vocab_path = os.path.join(working_directory, "vocab%d.enc" % enc_vocabulary_size)
  dec_vocab_path = os.path.join(working_directory, "vocab%d.dec" % dec_vocabulary_size)
@@ -143,4 +147,4 @@ def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_
  data_to_token_ids(test_enc, enc_dev_ids_path, enc_vocab_path, tokenizer)
  data_to_token_ids(test_dec, dec_dev_ids_path, dec_vocab_path, tokenizer)
 
- return (enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path)
+ return enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path

.gitignore

@@ -1,2 +1,5 @@
 data/
+working_dir/
 *.pyc
+.venv/
+.idea/

README.md

@@ -20,22 +20,81 @@ Use [pip](https://pypi.python.org/pypi/pip) to install any missing dependencies
 Usage
 ===========
 
-To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
+Create venv & install dependencies:
+
+> Using viritualenv here to be compatable with python2, install with "pip install virtualenv"
+
+```
+# create venv
+python -m virtualenv .venv
+
+# enter venv (assuming macos/linux)
+source .venv/bin/activate
+
+# install requirements
+pip install -r requirements.txt
+```
+
+Training
+--------------------
+
+0. Create directories
+ ```
+ mkdir working_dir
+ mkdir data
+ ```
+
+1. Download the [Cornell Movie Dialogue dataset](https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html) and place the unzipped content in the `data/` directory.
+ ```
+ cd data
+ wget http://www.mpi-sws.org/~cristian/data/cornell_movie_dialogs_corpus.zip
+ unzip cornell_movie_dialogs_corpus.zip
+
+ # move to `data/` root
+ mv "corenell movie-dialogs corpus"/* .
+ ```
+
+2. Prepare data for training, in the `data/` directory, run the `prepare_data.py` script
+ ```
+ # move to project root
+ cd ..
+ python prepare_data.py
+ ```
+
+3. To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
 
 `mode = train`
 
-then run the code like so
+4. Start training, by running the code like so:
+
+ ``python execute.py``
+
+> There is no mechanism to stop training, you will need to 'ctrl-c' to stop training after a period of time.
+
+
+Test
+-------------
+
+1. To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+
+ `mode = test`
 
-``python execute.py``
+2. To test run the code like so:
 
-To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+ ```
+ python execute.py
 
-`mode = test`
+ >> Mode : test
 
-then run the code like so
+ Reading model parameters from working_dir/seq2seq.ckpt-10200
+ > 
+ ```
 
-``python execute.py``
+
 
+3. Confrim... 
+ - What does "Test do?"
+ - How to use it?
 
 Challenge
 ===========

data_utils.py

@@ -20,16 +20,16 @@
 
 import os
 import re
-
-from six.moves import urllib
+from io import open
+from collections import Counter
 
 from tensorflow.python.platform import gfile
 
 # Special vocabulary symbols - we always put them at the start.
-_PAD = b"_PAD"
-_GO = b"_GO"
-_EOS = b"_EOS"
-_UNK = b"_UNK"
+_PAD = "_PAD"
+_GO = "_GO"
+_EOS = "_EOS"
+_UNK = "_UNK"
 _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
 
 PAD_ID = 0
@@ -38,93 +38,97 @@
 UNK_ID = 3
 
 # Regular expressions used to tokenize.
-_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
-_DIGIT_RE = re.compile(br"\d")
+_WORD_SPLIT = re.compile("([.,!?\"':;)(])")
+_DIGIT_RE = re.compile(r"\d")
 
+CORNELL_MOVIE_CORPUS_ENCODING = 'ISO-8859-2'
 
 
 def basic_tokenizer(sentence):
- """Very basic tokenizer: split the sentence into a list of tokens."""
- words = []
- for space_separated_fragment in sentence.strip().split():
- words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
- return [w for w in words if w]
+ """Very basic tokenizer: split the sentence into a list of tokens."""
+ all_words = []
+ for space_separated_fragment in sentence.strip().split():
+ words = re.split(_WORD_SPLIT, space_separated_fragment)
+ for word in words:
+ if word:
+ all_words.append(word)
+ return all_words
 
 
 def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(vocabulary_path):
- print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ if not tokenizer:
+ tokenizer = basic_tokenizer
+
+ if not os.path.exists(vocabulary_path):
+ print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ vocab = Counter()
+ with open(data_path, 'rt', encoding='utf8') as f:
+ for counter, sentence in enumerate(f, 1):
+ if counter % 100000 == 0:
+ print(" processing line %d" % counter)
+ tokens = tokenizer(sentence)
+ for w in tokens:
+ if normalize_digits:
+ word = re.sub(_DIGIT_RE, '0', w)
+ else:
+ word = w
+ vocab[word] += 1
+
+ vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
+ print('>> Full Vocabulary Size :', len(vocab_list))
+ if len(vocab_list) > max_vocabulary_size:
+ vocab_list = vocab_list[:max_vocabulary_size]
+ print('>>>> Vocab Truncated to: {}'.format(max_vocabulary_size))
+ with open(vocabulary_path, 'wt', encoding='utf8') as vocab_file:
+ for w in vocab_list:
+ vocab_file.write(w + '\n')
+
+
+def initialize_vocabulary(vocabulary_path, encoding=CORNELL_MOVIE_CORPUS_ENCODING):
  vocab = {}
- with gfile.GFile(data_path, mode="rb") as f:
- counter = 0
- for line in f:
- counter += 1
- if counter % 100000 == 0:
- print(" processing line %d" % counter)
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1
- else:
- vocab[word] = 1
- vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
- print('>> Full Vocabulary Size :',len(vocab_list))
- if len(vocab_list) > max_vocabulary_size:
- vocab_list = vocab_list[:max_vocabulary_size]
- with gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
- for w in vocab_list:
- vocab_file.write(w + b"\n")
-
-
-def initialize_vocabulary(vocabulary_path):
-
- if gfile.Exists(vocabulary_path):
  rev_vocab = []
- with gfile.GFile(vocabulary_path, mode="rb") as f:
- rev_vocab.extend(f.readlines())
- rev_vocab = [line.strip() for line in rev_vocab]
- vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
+ if gfile.Exists(vocabulary_path):
+ with open(vocabulary_path, 'rt', encoding=encoding) as f:
+ for index, line in enumerate(f, 1):
+ element = line.strip()
+ rev_vocab.append(element)
+ vocab[element] = index
+ assert len(vocab) == len(rev_vocab)
+ if not (vocab and rev_vocab):
+ raise ValueError('File empty: {}'.format(vocabulary_path))
+ else:
+ raise ValueError("Vocabulary file %s not found.", vocabulary_path)
  return vocab, rev_vocab
- else:
- raise ValueError("Vocabulary file %s not found.", vocabulary_path)
 
 
 def sentence_to_token_ids(sentence, vocabulary, tokenizer=None, normalize_digits=True):
-
-if tokenizer:
+ if not tokenizer:
+tokenizer = basic_tokenizer
  words = tokenizer(sentence)
- else:
- words = basic_tokenizer(sentence)
- if not normalize_digits:
- return [vocabulary.get(w, UNK_ID) for w in words]
- # Normalize digits by 0 before looking words up in the vocabulary.
- return [vocabulary.get(re.sub(_DIGIT_RE, b"0", w), UNK_ID) for w in words]
+ if not normalize_digits:
+ return [vocabulary.get(w, UNK_ID) for w in words]
+ # Normalize digits by 0 before looking words up in the vocabulary.
+ return [vocabulary.get(re.sub(_DIGIT_RE, '0', w), UNK_ID) for w in words]
 
 
 def data_to_token_ids(data_path, target_path, vocabulary_path,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(target_path):
- print("Tokenizing data in %s" % data_path)
- vocab, _ = initialize_vocabulary(vocabulary_path)
- with gfile.GFile(data_path, mode="rb") as data_file:
- with gfile.GFile(target_path, mode="w") as tokens_file:
- counter = 0
- for line in data_file:
- counter += 1
- if counter % 100000 == 0:
- print(" tokenizing line %d" % counter)
- token_ids = sentence_to_token_ids(line, vocab, tokenizer,
- normalize_digits)
- tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
-
-
-
-def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size, dec_vocabulary_size, tokenizer=None):
-
+ if not gfile.Exists(target_path):
+ print("Tokenizing data in %s" % data_path)
+ vocab, _ = initialize_vocabulary(vocabulary_path)
+ with gfile.GFile(data_path, mode="rb") as data_file:
+ with gfile.GFile(target_path, mode="w") as tokens_file:
+ for counter, line in enumerate(data_file, 1):
+ if counter % 100000 == 0:
+ print(" tokenizing line %d" % counter)
+ token_ids = sentence_to_token_ids(line, vocab, tokenizer,
+ normalize_digits)
+ tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
+
+
+def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size,
+ dec_vocabulary_size, tokenizer=None):
  # Create vocabularies of the appropriate sizes.
  enc_vocab_path = os.path.join(working_directory, "vocab%d.enc" % enc_vocabulary_size)
  dec_vocab_path = os.path.join(working_directory, "vocab%d.dec" % dec_vocabulary_size)
@@ -143,4 +147,4 @@ def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_
  data_to_token_ids(test_enc, enc_dev_ids_path, enc_vocab_path, tokenizer)
  data_to_token_ids(test_dec, dec_dev_ids_path, dec_vocab_path, tokenizer)
 
- return (enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path)
+ return enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path

.gitignore

@@ -1,2 +1,5 @@
 data/
+working_dir/
 *.pyc
+.venv/
+.idea/

README.md

@@ -20,22 +20,81 @@ Use [pip](https://pypi.python.org/pypi/pip) to install any missing dependencies
 Usage
 ===========
 
-To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
+Create venv & install dependencies:
+
+> Using viritualenv here to be compatable with python2, install with "pip install virtualenv"
+
+```
+# create venv
+python -m virtualenv .venv
+
+# enter venv (assuming macos/linux)
+source .venv/bin/activate
+
+# install requirements
+pip install -r requirements.txt
+```
+
+Training
+--------------------
+
+0. Create directories
+ ```
+ mkdir working_dir
+ mkdir data
+ ```
+
+1. Download the [Cornell Movie Dialogue dataset](https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html) and place the unzipped content in the `data/` directory.
+ ```
+ cd data
+ wget http://www.mpi-sws.org/~cristian/data/cornell_movie_dialogs_corpus.zip
+ unzip cornell_movie_dialogs_corpus.zip
+
+ # move to `data/` root
+ mv "corenell movie-dialogs corpus"/* .
+ ```
+
+2. Prepare data for training, in the `data/` directory, run the `prepare_data.py` script
+ ```
+ # move to project root
+ cd ..
+ python prepare_data.py
+ ```
+
+3. To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
 
 `mode = train`
 
-then run the code like so
+4. Start training, by running the code like so:
+
+ ``python execute.py``
+
+> There is no mechanism to stop training, you will need to 'ctrl-c' to stop training after a period of time.
+
+
+Test
+-------------
+
+1. To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+
+ `mode = test`
 
-``python execute.py``
+2. To test run the code like so:
 
-To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+ ```
+ python execute.py
 
-`mode = test`
+ >> Mode : test
 
-then run the code like so
+ Reading model parameters from working_dir/seq2seq.ckpt-10200
+ > 
+ ```
 
-``python execute.py``
+
 
+3. Confrim... 
+ - What does "Test do?"
+ - How to use it?
 
 Challenge
 ===========

data_utils.py

@@ -20,16 +20,16 @@
 
 import os
 import re
-
-from six.moves import urllib
+from io import open
+from collections import Counter
 
 from tensorflow.python.platform import gfile
 
 # Special vocabulary symbols - we always put them at the start.
-_PAD = b"_PAD"
-_GO = b"_GO"
-_EOS = b"_EOS"
-_UNK = b"_UNK"
+_PAD = "_PAD"
+_GO = "_GO"
+_EOS = "_EOS"
+_UNK = "_UNK"
 _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
 
 PAD_ID = 0
@@ -38,93 +38,97 @@
 UNK_ID = 3
 
 # Regular expressions used to tokenize.
-_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
-_DIGIT_RE = re.compile(br"\d")
+_WORD_SPLIT = re.compile("([.,!?\"':;)(])")
+_DIGIT_RE = re.compile(r"\d")
 
+CORNELL_MOVIE_CORPUS_ENCODING = 'ISO-8859-2'
 
 
 def basic_tokenizer(sentence):
- """Very basic tokenizer: split the sentence into a list of tokens."""
- words = []
- for space_separated_fragment in sentence.strip().split():
- words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
- return [w for w in words if w]
+ """Very basic tokenizer: split the sentence into a list of tokens."""
+ all_words = []
+ for space_separated_fragment in sentence.strip().split():
+ words = re.split(_WORD_SPLIT, space_separated_fragment)
+ for word in words:
+ if word:
+ all_words.append(word)
+ return all_words
 
 
 def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(vocabulary_path):
- print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ if not tokenizer:
+ tokenizer = basic_tokenizer
+
+ if not os.path.exists(vocabulary_path):
+ print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ vocab = Counter()
+ with open(data_path, 'rt', encoding='utf8') as f:
+ for counter, sentence in enumerate(f, 1):
+ if counter % 100000 == 0:
+ print(" processing line %d" % counter)
+ tokens = tokenizer(sentence)
+ for w in tokens:
+ if normalize_digits:
+ word = re.sub(_DIGIT_RE, '0', w)
+ else:
+ word = w
+ vocab[word] += 1
+
+ vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
+ print('>> Full Vocabulary Size :', len(vocab_list))
+ if len(vocab_list) > max_vocabulary_size:
+ vocab_list = vocab_list[:max_vocabulary_size]
+ print('>>>> Vocab Truncated to: {}'.format(max_vocabulary_size))
+ with open(vocabulary_path, 'wt', encoding='utf8') as vocab_file:
+ for w in vocab_list:
+ vocab_file.write(w + '\n')
+
+
+def initialize_vocabulary(vocabulary_path, encoding=CORNELL_MOVIE_CORPUS_ENCODING):
  vocab = {}
- with gfile.GFile(data_path, mode="rb") as f:
- counter = 0
- for line in f:
- counter += 1
- if counter % 100000 == 0:
- print(" processing line %d" % counter)
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1
- else:
- vocab[word] = 1
- vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
- print('>> Full Vocabulary Size :',len(vocab_list))
- if len(vocab_list) > max_vocabulary_size:
- vocab_list = vocab_list[:max_vocabulary_size]
- with gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
- for w in vocab_list:
- vocab_file.write(w + b"\n")
-
-
-def initialize_vocabulary(vocabulary_path):
-
- if gfile.Exists(vocabulary_path):
  rev_vocab = []
- with gfile.GFile(vocabulary_path, mode="rb") as f:
- rev_vocab.extend(f.readlines())
- rev_vocab = [line.strip() for line in rev_vocab]
- vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
+ if gfile.Exists(vocabulary_path):
+ with open(vocabulary_path, 'rt', encoding=encoding) as f:
+ for index, line in enumerate(f, 1):
+ element = line.strip()
+ rev_vocab.append(element)
+ vocab[element] = index
+ assert len(vocab) == len(rev_vocab)
+ if not (vocab and rev_vocab):
+ raise ValueError('File empty: {}'.format(vocabulary_path))
+ else:
+ raise ValueError("Vocabulary file %s not found.", vocabulary_path)
  return vocab, rev_vocab
- else:
- raise ValueError("Vocabulary file %s not found.", vocabulary_path)
 
 
 def sentence_to_token_ids(sentence, vocabulary, tokenizer=None, normalize_digits=True):
-
-if tokenizer:
+ if not tokenizer:
+tokenizer = basic_tokenizer
  words = tokenizer(sentence)
- else:
- words = basic_tokenizer(sentence)
- if not normalize_digits:
- return [vocabulary.get(w, UNK_ID) for w in words]
- # Normalize digits by 0 before looking words up in the vocabulary.
- return [vocabulary.get(re.sub(_DIGIT_RE, b"0", w), UNK_ID) for w in words]
+ if not normalize_digits:
+ return [vocabulary.get(w, UNK_ID) for w in words]
+ # Normalize digits by 0 before looking words up in the vocabulary.
+ return [vocabulary.get(re.sub(_DIGIT_RE, '0', w), UNK_ID) for w in words]
 
 
 def data_to_token_ids(data_path, target_path, vocabulary_path,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(target_path):
- print("Tokenizing data in %s" % data_path)
- vocab, _ = initialize_vocabulary(vocabulary_path)
- with gfile.GFile(data_path, mode="rb") as data_file:
- with gfile.GFile(target_path, mode="w") as tokens_file:
- counter = 0
- for line in data_file:
- counter += 1
- if counter % 100000 == 0:
- print(" tokenizing line %d" % counter)
- token_ids = sentence_to_token_ids(line, vocab, tokenizer,
- normalize_digits)
- tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
-
-
-
-def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size, dec_vocabulary_size, tokenizer=None):
-
+ if not gfile.Exists(target_path):
+ print("Tokenizing data in %s" % data_path)
+ vocab, _ = initialize_vocabulary(vocabulary_path)
+ with gfile.GFile(data_path, mode="rb") as data_file:
+ with gfile.GFile(target_path, mode="w") as tokens_file:
+ for counter, line in enumerate(data_file, 1):
+ if counter % 100000 == 0:
+ print(" tokenizing line %d" % counter)
+ token_ids = sentence_to_token_ids(line, vocab, tokenizer,
+ normalize_digits)
+ tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
+
+
+def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size,
+ dec_vocabulary_size, tokenizer=None):
  # Create vocabularies of the appropriate sizes.
  enc_vocab_path = os.path.join(working_directory, "vocab%d.enc" % enc_vocabulary_size)
  dec_vocab_path = os.path.join(working_directory, "vocab%d.dec" % dec_vocabulary_size)
@@ -143,4 +147,4 @@ def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_
  data_to_token_ids(test_enc, enc_dev_ids_path, enc_vocab_path, tokenizer)
  data_to_token_ids(test_dec, dec_dev_ids_path, dec_vocab_path, tokenizer)
 
- return (enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path)
+ return enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path

.gitignore

@@ -1,2 +1,5 @@
 data/
+working_dir/
 *.pyc
+.venv/
+.idea/

README.md

@@ -20,22 +20,81 @@ Use [pip](https://pypi.python.org/pypi/pip) to install any missing dependencies
 Usage
 ===========
 
-To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
+Create venv & install dependencies:
+
+> Using viritualenv here to be compatable with python2, install with "pip install virtualenv"
+
+```
+# create venv
+python -m virtualenv .venv
+
+# enter venv (assuming macos/linux)
+source .venv/bin/activate
+
+# install requirements
+pip install -r requirements.txt
+```
+
+Training
+--------------------
+
+0. Create directories
+ ```
+ mkdir working_dir
+ mkdir data
+ ```
+
+1. Download the [Cornell Movie Dialogue dataset](https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html) and place the unzipped content in the `data/` directory.
+ ```
+ cd data
+ wget http://www.mpi-sws.org/~cristian/data/cornell_movie_dialogs_corpus.zip
+ unzip cornell_movie_dialogs_corpus.zip
+
+ # move to `data/` root
+ mv "corenell movie-dialogs corpus"/* .
+ ```
+
+2. Prepare data for training, in the `data/` directory, run the `prepare_data.py` script
+ ```
+ # move to project root
+ cd ..
+ python prepare_data.py
+ ```
+
+3. To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
 
 `mode = train`
 
-then run the code like so
+4. Start training, by running the code like so:
+
+ ``python execute.py``
+
+> There is no mechanism to stop training, you will need to 'ctrl-c' to stop training after a period of time.
+
+
+Test
+-------------
+
+1. To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+
+ `mode = test`
 
-``python execute.py``
+2. To test run the code like so:
 
-To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+ ```
+ python execute.py
 
-`mode = test`
+ >> Mode : test
 
-then run the code like so
+ Reading model parameters from working_dir/seq2seq.ckpt-10200
+ > 
+ ```
 
-``python execute.py``
+
 
+3. Confrim... 
+ - What does "Test do?"
+ - How to use it?
 
 Challenge
 ===========

data_utils.py

@@ -20,16 +20,16 @@
 
 import os
 import re
-
-from six.moves import urllib
+from io import open
+from collections import Counter
 
 from tensorflow.python.platform import gfile
 
 # Special vocabulary symbols - we always put them at the start.
-_PAD = b"_PAD"
-_GO = b"_GO"
-_EOS = b"_EOS"
-_UNK = b"_UNK"
+_PAD = "_PAD"
+_GO = "_GO"
+_EOS = "_EOS"
+_UNK = "_UNK"
 _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
 
 PAD_ID = 0
@@ -38,93 +38,97 @@
 UNK_ID = 3
 
 # Regular expressions used to tokenize.
-_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
-_DIGIT_RE = re.compile(br"\d")
+_WORD_SPLIT = re.compile("([.,!?\"':;)(])")
+_DIGIT_RE = re.compile(r"\d")
 
+CORNELL_MOVIE_CORPUS_ENCODING = 'ISO-8859-2'
 
 
 def basic_tokenizer(sentence):
- """Very basic tokenizer: split the sentence into a list of tokens."""
- words = []
- for space_separated_fragment in sentence.strip().split():
- words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
- return [w for w in words if w]
+ """Very basic tokenizer: split the sentence into a list of tokens."""
+ all_words = []
+ for space_separated_fragment in sentence.strip().split():
+ words = re.split(_WORD_SPLIT, space_separated_fragment)
+ for word in words:
+ if word:
+ all_words.append(word)
+ return all_words
 
 
 def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(vocabulary_path):
- print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ if not tokenizer:
+ tokenizer = basic_tokenizer
+
+ if not os.path.exists(vocabulary_path):
+ print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ vocab = Counter()
+ with open(data_path, 'rt', encoding='utf8') as f:
+ for counter, sentence in enumerate(f, 1):
+ if counter % 100000 == 0:
+ print(" processing line %d" % counter)
+ tokens = tokenizer(sentence)
+ for w in tokens:
+ if normalize_digits:
+ word = re.sub(_DIGIT_RE, '0', w)
+ else:
+ word = w
+ vocab[word] += 1
+
+ vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
+ print('>> Full Vocabulary Size :', len(vocab_list))
+ if len(vocab_list) > max_vocabulary_size:
+ vocab_list = vocab_list[:max_vocabulary_size]
+ print('>>>> Vocab Truncated to: {}'.format(max_vocabulary_size))
+ with open(vocabulary_path, 'wt', encoding='utf8') as vocab_file:
+ for w in vocab_list:
+ vocab_file.write(w + '\n')
+
+
+def initialize_vocabulary(vocabulary_path, encoding=CORNELL_MOVIE_CORPUS_ENCODING):
  vocab = {}
- with gfile.GFile(data_path, mode="rb") as f:
- counter = 0
- for line in f:
- counter += 1
- if counter % 100000 == 0:
- print(" processing line %d" % counter)
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1
- else:
- vocab[word] = 1
- vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
- print('>> Full Vocabulary Size :',len(vocab_list))
- if len(vocab_list) > max_vocabulary_size:
- vocab_list = vocab_list[:max_vocabulary_size]
- with gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
- for w in vocab_list:
- vocab_file.write(w + b"\n")
-
-
-def initialize_vocabulary(vocabulary_path):
-
- if gfile.Exists(vocabulary_path):
  rev_vocab = []
- with gfile.GFile(vocabulary_path, mode="rb") as f:
- rev_vocab.extend(f.readlines())
- rev_vocab = [line.strip() for line in rev_vocab]
- vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
+ if gfile.Exists(vocabulary_path):
+ with open(vocabulary_path, 'rt', encoding=encoding) as f:
+ for index, line in enumerate(f, 1):
+ element = line.strip()
+ rev_vocab.append(element)
+ vocab[element] = index
+ assert len(vocab) == len(rev_vocab)
+ if not (vocab and rev_vocab):
+ raise ValueError('File empty: {}'.format(vocabulary_path))
+ else:
+ raise ValueError("Vocabulary file %s not found.", vocabulary_path)
  return vocab, rev_vocab
- else:
- raise ValueError("Vocabulary file %s not found.", vocabulary_path)
 
 
 def sentence_to_token_ids(sentence, vocabulary, tokenizer=None, normalize_digits=True):
-
-if tokenizer:
+ if not tokenizer:
+tokenizer = basic_tokenizer
  words = tokenizer(sentence)
- else:
- words = basic_tokenizer(sentence)
- if not normalize_digits:
- return [vocabulary.get(w, UNK_ID) for w in words]
- # Normalize digits by 0 before looking words up in the vocabulary.
- return [vocabulary.get(re.sub(_DIGIT_RE, b"0", w), UNK_ID) for w in words]
+ if not normalize_digits:
+ return [vocabulary.get(w, UNK_ID) for w in words]
+ # Normalize digits by 0 before looking words up in the vocabulary.
+ return [vocabulary.get(re.sub(_DIGIT_RE, '0', w), UNK_ID) for w in words]
 
 
 def data_to_token_ids(data_path, target_path, vocabulary_path,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(target_path):
- print("Tokenizing data in %s" % data_path)
- vocab, _ = initialize_vocabulary(vocabulary_path)
- with gfile.GFile(data_path, mode="rb") as data_file:
- with gfile.GFile(target_path, mode="w") as tokens_file:
- counter = 0
- for line in data_file:
- counter += 1
- if counter % 100000 == 0:
- print(" tokenizing line %d" % counter)
- token_ids = sentence_to_token_ids(line, vocab, tokenizer,
- normalize_digits)
- tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
-
-
-
-def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size, dec_vocabulary_size, tokenizer=None):
-
+ if not gfile.Exists(target_path):
+ print("Tokenizing data in %s" % data_path)
+ vocab, _ = initialize_vocabulary(vocabulary_path)
+ with gfile.GFile(data_path, mode="rb") as data_file:
+ with gfile.GFile(target_path, mode="w") as tokens_file:
+ for counter, line in enumerate(data_file, 1):
+ if counter % 100000 == 0:
+ print(" tokenizing line %d" % counter)
+ token_ids = sentence_to_token_ids(line, vocab, tokenizer,
+ normalize_digits)
+ tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
+
+
+def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size,
+ dec_vocabulary_size, tokenizer=None):
  # Create vocabularies of the appropriate sizes.
  enc_vocab_path = os.path.join(working_directory, "vocab%d.enc" % enc_vocabulary_size)
  dec_vocab_path = os.path.join(working_directory, "vocab%d.dec" % dec_vocabulary_size)
@@ -143,4 +147,4 @@ def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_
  data_to_token_ids(test_enc, enc_dev_ids_path, enc_vocab_path, tokenizer)
  data_to_token_ids(test_dec, dec_dev_ids_path, dec_vocab_path, tokenizer)
 
- return (enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path)
+ return enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path

.gitignore

@@ -1,2 +1,5 @@
 data/
+working_dir/
 *.pyc
+.venv/
+.idea/

README.md

@@ -20,22 +20,81 @@ Use [pip](https://pypi.python.org/pypi/pip) to install any missing dependencies
 Usage
 ===========
 
-To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
+Create venv & install dependencies:
+
+> Using viritualenv here to be compatable with python2, install with "pip install virtualenv"
+
+```
+# create venv
+python -m virtualenv .venv
+
+# enter venv (assuming macos/linux)
+source .venv/bin/activate
+
+# install requirements
+pip install -r requirements.txt
+```
+
+Training
+--------------------
+
+0. Create directories
+ ```
+ mkdir working_dir
+ mkdir data
+ ```
+
+1. Download the [Cornell Movie Dialogue dataset](https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html) and place the unzipped content in the `data/` directory.
+ ```
+ cd data
+ wget http://www.mpi-sws.org/~cristian/data/cornell_movie_dialogs_corpus.zip
+ unzip cornell_movie_dialogs_corpus.zip
+
+ # move to `data/` root
+ mv "corenell movie-dialogs corpus"/* .
+ ```
+
+2. Prepare data for training, in the `data/` directory, run the `prepare_data.py` script
+ ```
+ # move to project root
+ cd ..
+ python prepare_data.py
+ ```
+
+3. To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
 
 `mode = train`
 
-then run the code like so
+4. Start training, by running the code like so:
+
+ ``python execute.py``
+
+> There is no mechanism to stop training, you will need to 'ctrl-c' to stop training after a period of time.
+
+
+Test
+-------------
+
+1. To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+
+ `mode = test`
 
-``python execute.py``
+2. To test run the code like so:
 
-To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+ ```
+ python execute.py
 
-`mode = test`
+ >> Mode : test
 
-then run the code like so
+ Reading model parameters from working_dir/seq2seq.ckpt-10200
+ > 
+ ```
 
-``python execute.py``
+
 
+3. Confrim... 
+ - What does "Test do?"
+ - How to use it?
 
 Challenge
 ===========

data_utils.py

@@ -20,16 +20,16 @@
 
 import os
 import re
-
-from six.moves import urllib
+from io import open
+from collections import Counter
 
 from tensorflow.python.platform import gfile
 
 # Special vocabulary symbols - we always put them at the start.
-_PAD = b"_PAD"
-_GO = b"_GO"
-_EOS = b"_EOS"
-_UNK = b"_UNK"
+_PAD = "_PAD"
+_GO = "_GO"
+_EOS = "_EOS"
+_UNK = "_UNK"
 _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
 
 PAD_ID = 0
@@ -38,93 +38,97 @@
 UNK_ID = 3
 
 # Regular expressions used to tokenize.
-_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
-_DIGIT_RE = re.compile(br"\d")
+_WORD_SPLIT = re.compile("([.,!?\"':;)(])")
+_DIGIT_RE = re.compile(r"\d")
 
+CORNELL_MOVIE_CORPUS_ENCODING = 'ISO-8859-2'
 
 
 def basic_tokenizer(sentence):
- """Very basic tokenizer: split the sentence into a list of tokens."""
- words = []
- for space_separated_fragment in sentence.strip().split():
- words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
- return [w for w in words if w]
+ """Very basic tokenizer: split the sentence into a list of tokens."""
+ all_words = []
+ for space_separated_fragment in sentence.strip().split():
+ words = re.split(_WORD_SPLIT, space_separated_fragment)
+ for word in words:
+ if word:
+ all_words.append(word)
+ return all_words
 
 
 def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(vocabulary_path):
- print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ if not tokenizer:
+ tokenizer = basic_tokenizer
+
+ if not os.path.exists(vocabulary_path):
+ print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ vocab = Counter()
+ with open(data_path, 'rt', encoding='utf8') as f:
+ for counter, sentence in enumerate(f, 1):
+ if counter % 100000 == 0:
+ print(" processing line %d" % counter)
+ tokens = tokenizer(sentence)
+ for w in tokens:
+ if normalize_digits:
+ word = re.sub(_DIGIT_RE, '0', w)
+ else:
+ word = w
+ vocab[word] += 1
+
+ vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
+ print('>> Full Vocabulary Size :', len(vocab_list))
+ if len(vocab_list) > max_vocabulary_size:
+ vocab_list = vocab_list[:max_vocabulary_size]
+ print('>>>> Vocab Truncated to: {}'.format(max_vocabulary_size))
+ with open(vocabulary_path, 'wt', encoding='utf8') as vocab_file:
+ for w in vocab_list:
+ vocab_file.write(w + '\n')
+
+
+def initialize_vocabulary(vocabulary_path, encoding=CORNELL_MOVIE_CORPUS_ENCODING):
  vocab = {}
- with gfile.GFile(data_path, mode="rb") as f:
- counter = 0
- for line in f:
- counter += 1
- if counter % 100000 == 0:
- print(" processing line %d" % counter)
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1
- else:
- vocab[word] = 1
- vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
- print('>> Full Vocabulary Size :',len(vocab_list))
- if len(vocab_list) > max_vocabulary_size:
- vocab_list = vocab_list[:max_vocabulary_size]
- with gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
- for w in vocab_list:
- vocab_file.write(w + b"\n")
-
-
-def initialize_vocabulary(vocabulary_path):
-
- if gfile.Exists(vocabulary_path):
  rev_vocab = []
- with gfile.GFile(vocabulary_path, mode="rb") as f:
- rev_vocab.extend(f.readlines())
- rev_vocab = [line.strip() for line in rev_vocab]
- vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
+ if gfile.Exists(vocabulary_path):
+ with open(vocabulary_path, 'rt', encoding=encoding) as f:
+ for index, line in enumerate(f, 1):
+ element = line.strip()
+ rev_vocab.append(element)
+ vocab[element] = index
+ assert len(vocab) == len(rev_vocab)
+ if not (vocab and rev_vocab):
+ raise ValueError('File empty: {}'.format(vocabulary_path))
+ else:
+ raise ValueError("Vocabulary file %s not found.", vocabulary_path)
  return vocab, rev_vocab
- else:
- raise ValueError("Vocabulary file %s not found.", vocabulary_path)
 
 
 def sentence_to_token_ids(sentence, vocabulary, tokenizer=None, normalize_digits=True):
-
-if tokenizer:
+ if not tokenizer:
+tokenizer = basic_tokenizer
  words = tokenizer(sentence)
- else:
- words = basic_tokenizer(sentence)
- if not normalize_digits:
- return [vocabulary.get(w, UNK_ID) for w in words]
- # Normalize digits by 0 before looking words up in the vocabulary.
- return [vocabulary.get(re.sub(_DIGIT_RE, b"0", w), UNK_ID) for w in words]
+ if not normalize_digits:
+ return [vocabulary.get(w, UNK_ID) for w in words]
+ # Normalize digits by 0 before looking words up in the vocabulary.
+ return [vocabulary.get(re.sub(_DIGIT_RE, '0', w), UNK_ID) for w in words]
 
 
 def data_to_token_ids(data_path, target_path, vocabulary_path,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(target_path):
- print("Tokenizing data in %s" % data_path)
- vocab, _ = initialize_vocabulary(vocabulary_path)
- with gfile.GFile(data_path, mode="rb") as data_file:
- with gfile.GFile(target_path, mode="w") as tokens_file:
- counter = 0
- for line in data_file:
- counter += 1
- if counter % 100000 == 0:
- print(" tokenizing line %d" % counter)
- token_ids = sentence_to_token_ids(line, vocab, tokenizer,
- normalize_digits)
- tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
-
-
-
-def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size, dec_vocabulary_size, tokenizer=None):
-
+ if not gfile.Exists(target_path):
+ print("Tokenizing data in %s" % data_path)
+ vocab, _ = initialize_vocabulary(vocabulary_path)
+ with gfile.GFile(data_path, mode="rb") as data_file:
+ with gfile.GFile(target_path, mode="w") as tokens_file:
+ for counter, line in enumerate(data_file, 1):
+ if counter % 100000 == 0:
+ print(" tokenizing line %d" % counter)
+ token_ids = sentence_to_token_ids(line, vocab, tokenizer,
+ normalize_digits)
+ tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
+
+
+def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size,
+ dec_vocabulary_size, tokenizer=None):
  # Create vocabularies of the appropriate sizes.
  enc_vocab_path = os.path.join(working_directory, "vocab%d.enc" % enc_vocabulary_size)
  dec_vocab_path = os.path.join(working_directory, "vocab%d.dec" % dec_vocabulary_size)
@@ -143,4 +147,4 @@ def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_
  data_to_token_ids(test_enc, enc_dev_ids_path, enc_vocab_path, tokenizer)
  data_to_token_ids(test_dec, dec_dev_ids_path, dec_vocab_path, tokenizer)
 
- return (enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path)
+ return enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path

.gitignore

@@ -1,2 +1,5 @@
 data/
+working_dir/
 *.pyc
+.venv/
+.idea/

README.md

@@ -20,22 +20,81 @@ Use [pip](https://pypi.python.org/pypi/pip) to install any missing dependencies
 Usage
 ===========
 
-To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
+Create venv & install dependencies:
+
+> Using viritualenv here to be compatable with python2, install with "pip install virtualenv"
+
+```
+# create venv
+python -m virtualenv .venv
+
+# enter venv (assuming macos/linux)
+source .venv/bin/activate
+
+# install requirements
+pip install -r requirements.txt
+```
+
+Training
+--------------------
+
+0. Create directories
+ ```
+ mkdir working_dir
+ mkdir data
+ ```
+
+1. Download the [Cornell Movie Dialogue dataset](https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html) and place the unzipped content in the `data/` directory.
+ ```
+ cd data
+ wget http://www.mpi-sws.org/~cristian/data/cornell_movie_dialogs_corpus.zip
+ unzip cornell_movie_dialogs_corpus.zip
+
+ # move to `data/` root
+ mv "corenell movie-dialogs corpus"/* .
+ ```
+
+2. Prepare data for training, in the `data/` directory, run the `prepare_data.py` script
+ ```
+ # move to project root
+ cd ..
+ python prepare_data.py
+ ```
+
+3. To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
 
 `mode = train`
 
-then run the code like so
+4. Start training, by running the code like so:
+
+ ``python execute.py``
+
+> There is no mechanism to stop training, you will need to 'ctrl-c' to stop training after a period of time.
+
+
+Test
+-------------
+
+1. To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+
+ `mode = test`
 
-``python execute.py``
+2. To test run the code like so:
 
-To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+ ```
+ python execute.py
 
-`mode = test`
+ >> Mode : test
 
-then run the code like so
+ Reading model parameters from working_dir/seq2seq.ckpt-10200
+ > 
+ ```
 
-``python execute.py``
+
 
+3. Confrim... 
+ - What does "Test do?"
+ - How to use it?
 
 Challenge
 ===========

data_utils.py

@@ -20,16 +20,16 @@
 
 import os
 import re
-
-from six.moves import urllib
+from io import open
+from collections import Counter
 
 from tensorflow.python.platform import gfile
 
 # Special vocabulary symbols - we always put them at the start.
-_PAD = b"_PAD"
-_GO = b"_GO"
-_EOS = b"_EOS"
-_UNK = b"_UNK"
+_PAD = "_PAD"
+_GO = "_GO"
+_EOS = "_EOS"
+_UNK = "_UNK"
 _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
 
 PAD_ID = 0
@@ -38,93 +38,97 @@
 UNK_ID = 3
 
 # Regular expressions used to tokenize.
-_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
-_DIGIT_RE = re.compile(br"\d")
+_WORD_SPLIT = re.compile("([.,!?\"':;)(])")
+_DIGIT_RE = re.compile(r"\d")
 
+CORNELL_MOVIE_CORPUS_ENCODING = 'ISO-8859-2'
 
 
 def basic_tokenizer(sentence):
- """Very basic tokenizer: split the sentence into a list of tokens."""
- words = []
- for space_separated_fragment in sentence.strip().split():
- words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
- return [w for w in words if w]
+ """Very basic tokenizer: split the sentence into a list of tokens."""
+ all_words = []
+ for space_separated_fragment in sentence.strip().split():
+ words = re.split(_WORD_SPLIT, space_separated_fragment)
+ for word in words:
+ if word:
+ all_words.append(word)
+ return all_words
 
 
 def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(vocabulary_path):
- print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ if not tokenizer:
+ tokenizer = basic_tokenizer
+
+ if not os.path.exists(vocabulary_path):
+ print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ vocab = Counter()
+ with open(data_path, 'rt', encoding='utf8') as f:
+ for counter, sentence in enumerate(f, 1):
+ if counter % 100000 == 0:
+ print(" processing line %d" % counter)
+ tokens = tokenizer(sentence)
+ for w in tokens:
+ if normalize_digits:
+ word = re.sub(_DIGIT_RE, '0', w)
+ else:
+ word = w
+ vocab[word] += 1
+
+ vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
+ print('>> Full Vocabulary Size :', len(vocab_list))
+ if len(vocab_list) > max_vocabulary_size:
+ vocab_list = vocab_list[:max_vocabulary_size]
+ print('>>>> Vocab Truncated to: {}'.format(max_vocabulary_size))
+ with open(vocabulary_path, 'wt', encoding='utf8') as vocab_file:
+ for w in vocab_list:
+ vocab_file.write(w + '\n')
+
+
+def initialize_vocabulary(vocabulary_path, encoding=CORNELL_MOVIE_CORPUS_ENCODING):
  vocab = {}
- with gfile.GFile(data_path, mode="rb") as f:
- counter = 0
- for line in f:
- counter += 1
- if counter % 100000 == 0:
- print(" processing line %d" % counter)
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1
- else:
- vocab[word] = 1
- vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
- print('>> Full Vocabulary Size :',len(vocab_list))
- if len(vocab_list) > max_vocabulary_size:
- vocab_list = vocab_list[:max_vocabulary_size]
- with gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
- for w in vocab_list:
- vocab_file.write(w + b"\n")
-
-
-def initialize_vocabulary(vocabulary_path):
-
- if gfile.Exists(vocabulary_path):
  rev_vocab = []
- with gfile.GFile(vocabulary_path, mode="rb") as f:
- rev_vocab.extend(f.readlines())
- rev_vocab = [line.strip() for line in rev_vocab]
- vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
+ if gfile.Exists(vocabulary_path):
+ with open(vocabulary_path, 'rt', encoding=encoding) as f:
+ for index, line in enumerate(f, 1):
+ element = line.strip()
+ rev_vocab.append(element)
+ vocab[element] = index
+ assert len(vocab) == len(rev_vocab)
+ if not (vocab and rev_vocab):
+ raise ValueError('File empty: {}'.format(vocabulary_path))
+ else:
+ raise ValueError("Vocabulary file %s not found.", vocabulary_path)
  return vocab, rev_vocab
- else:
- raise ValueError("Vocabulary file %s not found.", vocabulary_path)
 
 
 def sentence_to_token_ids(sentence, vocabulary, tokenizer=None, normalize_digits=True):
-
-if tokenizer:
+ if not tokenizer:
+tokenizer = basic_tokenizer
  words = tokenizer(sentence)
- else:
- words = basic_tokenizer(sentence)
- if not normalize_digits:
- return [vocabulary.get(w, UNK_ID) for w in words]
- # Normalize digits by 0 before looking words up in the vocabulary.
- return [vocabulary.get(re.sub(_DIGIT_RE, b"0", w), UNK_ID) for w in words]
+ if not normalize_digits:
+ return [vocabulary.get(w, UNK_ID) for w in words]
+ # Normalize digits by 0 before looking words up in the vocabulary.
+ return [vocabulary.get(re.sub(_DIGIT_RE, '0', w), UNK_ID) for w in words]
 
 
 def data_to_token_ids(data_path, target_path, vocabulary_path,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(target_path):
- print("Tokenizing data in %s" % data_path)
- vocab, _ = initialize_vocabulary(vocabulary_path)
- with gfile.GFile(data_path, mode="rb") as data_file:
- with gfile.GFile(target_path, mode="w") as tokens_file:
- counter = 0
- for line in data_file:
- counter += 1
- if counter % 100000 == 0:
- print(" tokenizing line %d" % counter)
- token_ids = sentence_to_token_ids(line, vocab, tokenizer,
- normalize_digits)
- tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
-
-
-
-def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size, dec_vocabulary_size, tokenizer=None):
-
+ if not gfile.Exists(target_path):
+ print("Tokenizing data in %s" % data_path)
+ vocab, _ = initialize_vocabulary(vocabulary_path)
+ with gfile.GFile(data_path, mode="rb") as data_file:
+ with gfile.GFile(target_path, mode="w") as tokens_file:
+ for counter, line in enumerate(data_file, 1):
+ if counter % 100000 == 0:
+ print(" tokenizing line %d" % counter)
+ token_ids = sentence_to_token_ids(line, vocab, tokenizer,
+ normalize_digits)
+ tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
+
+
+def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size,
+ dec_vocabulary_size, tokenizer=None):
  # Create vocabularies of the appropriate sizes.
  enc_vocab_path = os.path.join(working_directory, "vocab%d.enc" % enc_vocabulary_size)
  dec_vocab_path = os.path.join(working_directory, "vocab%d.dec" % dec_vocabulary_size)
@@ -143,4 +147,4 @@ def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_
  data_to_token_ids(test_enc, enc_dev_ids_path, enc_vocab_path, tokenizer)
  data_to_token_ids(test_dec, dec_dev_ids_path, dec_vocab_path, tokenizer)
 
- return (enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path)
+ return enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path

.gitignore

@@ -1,2 +1,5 @@
 data/
+working_dir/
 *.pyc
+.venv/
+.idea/

README.md

@@ -20,22 +20,81 @@ Use [pip](https://pypi.python.org/pypi/pip) to install any missing dependencies
 Usage
 ===========
 
-To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
+Create venv & install dependencies:
+
+> Using viritualenv here to be compatable with python2, install with "pip install virtualenv"
+
+```
+# create venv
+python -m virtualenv .venv
+
+# enter venv (assuming macos/linux)
+source .venv/bin/activate
+
+# install requirements
+pip install -r requirements.txt
+```
+
+Training
+--------------------
+
+0. Create directories
+ ```
+ mkdir working_dir
+ mkdir data
+ ```
+
+1. Download the [Cornell Movie Dialogue dataset](https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html) and place the unzipped content in the `data/` directory.
+ ```
+ cd data
+ wget http://www.mpi-sws.org/~cristian/data/cornell_movie_dialogs_corpus.zip
+ unzip cornell_movie_dialogs_corpus.zip
+
+ # move to `data/` root
+ mv "corenell movie-dialogs corpus"/* .
+ ```
+
+2. Prepare data for training, in the `data/` directory, run the `prepare_data.py` script
+ ```
+ # move to project root
+ cd ..
+ python prepare_data.py
+ ```
+
+3. To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
 
 `mode = train`
 
-then run the code like so
+4. Start training, by running the code like so:
+
+ ``python execute.py``
+
+> There is no mechanism to stop training, you will need to 'ctrl-c' to stop training after a period of time.
+
+
+Test
+-------------
+
+1. To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+
+ `mode = test`
 
-``python execute.py``
+2. To test run the code like so:
 
-To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+ ```
+ python execute.py
 
-`mode = test`
+ >> Mode : test
 
-then run the code like so
+ Reading model parameters from working_dir/seq2seq.ckpt-10200
+ > 
+ ```
 
-``python execute.py``
+
 
+3. Confrim... 
+ - What does "Test do?"
+ - How to use it?
 
 Challenge
 ===========

data_utils.py

@@ -20,16 +20,16 @@
 
 import os
 import re
-
-from six.moves import urllib
+from io import open
+from collections import Counter
 
 from tensorflow.python.platform import gfile
 
 # Special vocabulary symbols - we always put them at the start.
-_PAD = b"_PAD"
-_GO = b"_GO"
-_EOS = b"_EOS"
-_UNK = b"_UNK"
+_PAD = "_PAD"
+_GO = "_GO"
+_EOS = "_EOS"
+_UNK = "_UNK"
 _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
 
 PAD_ID = 0
@@ -38,93 +38,97 @@
 UNK_ID = 3
 
 # Regular expressions used to tokenize.
-_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
-_DIGIT_RE = re.compile(br"\d")
+_WORD_SPLIT = re.compile("([.,!?\"':;)(])")
+_DIGIT_RE = re.compile(r"\d")
 
+CORNELL_MOVIE_CORPUS_ENCODING = 'ISO-8859-2'
 
 
 def basic_tokenizer(sentence):
- """Very basic tokenizer: split the sentence into a list of tokens."""
- words = []
- for space_separated_fragment in sentence.strip().split():
- words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
- return [w for w in words if w]
+ """Very basic tokenizer: split the sentence into a list of tokens."""
+ all_words = []
+ for space_separated_fragment in sentence.strip().split():
+ words = re.split(_WORD_SPLIT, space_separated_fragment)
+ for word in words:
+ if word:
+ all_words.append(word)
+ return all_words
 
 
 def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(vocabulary_path):
- print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ if not tokenizer:
+ tokenizer = basic_tokenizer
+
+ if not os.path.exists(vocabulary_path):
+ print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ vocab = Counter()
+ with open(data_path, 'rt', encoding='utf8') as f:
+ for counter, sentence in enumerate(f, 1):
+ if counter % 100000 == 0:
+ print(" processing line %d" % counter)
+ tokens = tokenizer(sentence)
+ for w in tokens:
+ if normalize_digits:
+ word = re.sub(_DIGIT_RE, '0', w)
+ else:
+ word = w
+ vocab[word] += 1
+
+ vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
+ print('>> Full Vocabulary Size :', len(vocab_list))
+ if len(vocab_list) > max_vocabulary_size:
+ vocab_list = vocab_list[:max_vocabulary_size]
+ print('>>>> Vocab Truncated to: {}'.format(max_vocabulary_size))
+ with open(vocabulary_path, 'wt', encoding='utf8') as vocab_file:
+ for w in vocab_list:
+ vocab_file.write(w + '\n')
+
+
+def initialize_vocabulary(vocabulary_path, encoding=CORNELL_MOVIE_CORPUS_ENCODING):
  vocab = {}
- with gfile.GFile(data_path, mode="rb") as f:
- counter = 0
- for line in f:
- counter += 1
- if counter % 100000 == 0:
- print(" processing line %d" % counter)
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1
- else:
- vocab[word] = 1
- vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
- print('>> Full Vocabulary Size :',len(vocab_list))
- if len(vocab_list) > max_vocabulary_size:
- vocab_list = vocab_list[:max_vocabulary_size]
- with gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
- for w in vocab_list:
- vocab_file.write(w + b"\n")
-
-
-def initialize_vocabulary(vocabulary_path):
-
- if gfile.Exists(vocabulary_path):
  rev_vocab = []
- with gfile.GFile(vocabulary_path, mode="rb") as f:
- rev_vocab.extend(f.readlines())
- rev_vocab = [line.strip() for line in rev_vocab]
- vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
+ if gfile.Exists(vocabulary_path):
+ with open(vocabulary_path, 'rt', encoding=encoding) as f:
+ for index, line in enumerate(f, 1):
+ element = line.strip()
+ rev_vocab.append(element)
+ vocab[element] = index
+ assert len(vocab) == len(rev_vocab)
+ if not (vocab and rev_vocab):
+ raise ValueError('File empty: {}'.format(vocabulary_path))
+ else:
+ raise ValueError("Vocabulary file %s not found.", vocabulary_path)
  return vocab, rev_vocab
- else:
- raise ValueError("Vocabulary file %s not found.", vocabulary_path)
 
 
 def sentence_to_token_ids(sentence, vocabulary, tokenizer=None, normalize_digits=True):
-
-if tokenizer:
+ if not tokenizer:
+tokenizer = basic_tokenizer
  words = tokenizer(sentence)
- else:
- words = basic_tokenizer(sentence)
- if not normalize_digits:
- return [vocabulary.get(w, UNK_ID) for w in words]
- # Normalize digits by 0 before looking words up in the vocabulary.
- return [vocabulary.get(re.sub(_DIGIT_RE, b"0", w), UNK_ID) for w in words]
+ if not normalize_digits:
+ return [vocabulary.get(w, UNK_ID) for w in words]
+ # Normalize digits by 0 before looking words up in the vocabulary.
+ return [vocabulary.get(re.sub(_DIGIT_RE, '0', w), UNK_ID) for w in words]
 
 
 def data_to_token_ids(data_path, target_path, vocabulary_path,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(target_path):
- print("Tokenizing data in %s" % data_path)
- vocab, _ = initialize_vocabulary(vocabulary_path)
- with gfile.GFile(data_path, mode="rb") as data_file:
- with gfile.GFile(target_path, mode="w") as tokens_file:
- counter = 0
- for line in data_file:
- counter += 1
- if counter % 100000 == 0:
- print(" tokenizing line %d" % counter)
- token_ids = sentence_to_token_ids(line, vocab, tokenizer,
- normalize_digits)
- tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
-
-
-
-def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size, dec_vocabulary_size, tokenizer=None):
-
+ if not gfile.Exists(target_path):
+ print("Tokenizing data in %s" % data_path)
+ vocab, _ = initialize_vocabulary(vocabulary_path)
+ with gfile.GFile(data_path, mode="rb") as data_file:
+ with gfile.GFile(target_path, mode="w") as tokens_file:
+ for counter, line in enumerate(data_file, 1):
+ if counter % 100000 == 0:
+ print(" tokenizing line %d" % counter)
+ token_ids = sentence_to_token_ids(line, vocab, tokenizer,
+ normalize_digits)
+ tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
+
+
+def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size,
+ dec_vocabulary_size, tokenizer=None):
  # Create vocabularies of the appropriate sizes.
  enc_vocab_path = os.path.join(working_directory, "vocab%d.enc" % enc_vocabulary_size)
  dec_vocab_path = os.path.join(working_directory, "vocab%d.dec" % dec_vocabulary_size)
@@ -143,4 +147,4 @@ def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_
  data_to_token_ids(test_enc, enc_dev_ids_path, enc_vocab_path, tokenizer)
  data_to_token_ids(test_dec, dec_dev_ids_path, dec_vocab_path, tokenizer)
 
- return (enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path)
+ return enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path

.gitignore

@@ -1,2 +1,5 @@
 data/
+working_dir/
 *.pyc
+.venv/
+.idea/

README.md

@@ -20,22 +20,81 @@ Use [pip](https://pypi.python.org/pypi/pip) to install any missing dependencies
 Usage
 ===========
 
-To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
+Create venv & install dependencies:
+
+> Using viritualenv here to be compatable with python2, install with "pip install virtualenv"
+
+```
+# create venv
+python -m virtualenv .venv
+
+# enter venv (assuming macos/linux)
+source .venv/bin/activate
+
+# install requirements
+pip install -r requirements.txt
+```
+
+Training
+--------------------
+
+0. Create directories
+ ```
+ mkdir working_dir
+ mkdir data
+ ```
+
+1. Download the [Cornell Movie Dialogue dataset](https://www.cs.cornell.edu/~cristian/Cornell_Movie-Dialogs_Corpus.html) and place the unzipped content in the `data/` directory.
+ ```
+ cd data
+ wget http://www.mpi-sws.org/~cristian/data/cornell_movie_dialogs_corpus.zip
+ unzip cornell_movie_dialogs_corpus.zip
+
+ # move to `data/` root
+ mv "corenell movie-dialogs corpus"/* .
+ ```
+
+2. Prepare data for training, in the `data/` directory, run the `prepare_data.py` script
+ ```
+ # move to project root
+ cd ..
+ python prepare_data.py
+ ```
+
+3. To train the bot, edit the `seq2seq.ini` file so that mode is set to train like so
 
 `mode = train`
 
-then run the code like so
+4. Start training, by running the code like so:
+
+ ``python execute.py``
+
+> There is no mechanism to stop training, you will need to 'ctrl-c' to stop training after a period of time.
+
+
+Test
+-------------
+
+1. To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+
+ `mode = test`
 
-``python execute.py``
+2. To test run the code like so:
 
-To test the bot during or after training, edit the `seq2seq.ini` file so that mode is set to test like so
+ ```
+ python execute.py
 
-`mode = test`
+ >> Mode : test
 
-then run the code like so
+ Reading model parameters from working_dir/seq2seq.ckpt-10200
+ > 
+ ```
 
-``python execute.py``
+
 
+3. Confrim... 
+ - What does "Test do?"
+ - How to use it?
 
 Challenge
 ===========

data_utils.py

@@ -20,16 +20,16 @@
 
 import os
 import re
-
-from six.moves import urllib
+from io import open
+from collections import Counter
 
 from tensorflow.python.platform import gfile
 
 # Special vocabulary symbols - we always put them at the start.
-_PAD = b"_PAD"
-_GO = b"_GO"
-_EOS = b"_EOS"
-_UNK = b"_UNK"
+_PAD = "_PAD"
+_GO = "_GO"
+_EOS = "_EOS"
+_UNK = "_UNK"
 _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
 
 PAD_ID = 0
@@ -38,93 +38,97 @@
 UNK_ID = 3
 
 # Regular expressions used to tokenize.
-_WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
-_DIGIT_RE = re.compile(br"\d")
+_WORD_SPLIT = re.compile("([.,!?\"':;)(])")
+_DIGIT_RE = re.compile(r"\d")
 
+CORNELL_MOVIE_CORPUS_ENCODING = 'ISO-8859-2'
 
 
 def basic_tokenizer(sentence):
- """Very basic tokenizer: split the sentence into a list of tokens."""
- words = []
- for space_separated_fragment in sentence.strip().split():
- words.extend(re.split(_WORD_SPLIT, space_separated_fragment))
- return [w for w in words if w]
+ """Very basic tokenizer: split the sentence into a list of tokens."""
+ all_words = []
+ for space_separated_fragment in sentence.strip().split():
+ words = re.split(_WORD_SPLIT, space_separated_fragment)
+ for word in words:
+ if word:
+ all_words.append(word)
+ return all_words
 
 
 def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(vocabulary_path):
- print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ if not tokenizer:
+ tokenizer = basic_tokenizer
+
+ if not os.path.exists(vocabulary_path):
+ print("Creating vocabulary %s from %s" % (vocabulary_path, data_path))
+ vocab = Counter()
+ with open(data_path, 'rt', encoding='utf8') as f:
+ for counter, sentence in enumerate(f, 1):
+ if counter % 100000 == 0:
+ print(" processing line %d" % counter)
+ tokens = tokenizer(sentence)
+ for w in tokens:
+ if normalize_digits:
+ word = re.sub(_DIGIT_RE, '0', w)
+ else:
+ word = w
+ vocab[word] += 1
+
+ vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
+ print('>> Full Vocabulary Size :', len(vocab_list))
+ if len(vocab_list) > max_vocabulary_size:
+ vocab_list = vocab_list[:max_vocabulary_size]
+ print('>>>> Vocab Truncated to: {}'.format(max_vocabulary_size))
+ with open(vocabulary_path, 'wt', encoding='utf8') as vocab_file:
+ for w in vocab_list:
+ vocab_file.write(w + '\n')
+
+
+def initialize_vocabulary(vocabulary_path, encoding=CORNELL_MOVIE_CORPUS_ENCODING):
  vocab = {}
- with gfile.GFile(data_path, mode="rb") as f:
- counter = 0
- for line in f:
- counter += 1
- if counter % 100000 == 0:
- print(" processing line %d" % counter)
- tokens = tokenizer(line) if tokenizer else basic_tokenizer(line)
- for w in tokens:
- word = re.sub(_DIGIT_RE, b"0", w) if normalize_digits else w
- if word in vocab:
- vocab[word] += 1
- else:
- vocab[word] = 1
- vocab_list = _START_VOCAB + sorted(vocab, key=vocab.get, reverse=True)
- print('>> Full Vocabulary Size :',len(vocab_list))
- if len(vocab_list) > max_vocabulary_size:
- vocab_list = vocab_list[:max_vocabulary_size]
- with gfile.GFile(vocabulary_path, mode="wb") as vocab_file:
- for w in vocab_list:
- vocab_file.write(w + b"\n")
-
-
-def initialize_vocabulary(vocabulary_path):
-
- if gfile.Exists(vocabulary_path):
  rev_vocab = []
- with gfile.GFile(vocabulary_path, mode="rb") as f:
- rev_vocab.extend(f.readlines())
- rev_vocab = [line.strip() for line in rev_vocab]
- vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
+ if gfile.Exists(vocabulary_path):
+ with open(vocabulary_path, 'rt', encoding=encoding) as f:
+ for index, line in enumerate(f, 1):
+ element = line.strip()
+ rev_vocab.append(element)
+ vocab[element] = index
+ assert len(vocab) == len(rev_vocab)
+ if not (vocab and rev_vocab):
+ raise ValueError('File empty: {}'.format(vocabulary_path))
+ else:
+ raise ValueError("Vocabulary file %s not found.", vocabulary_path)
  return vocab, rev_vocab
- else:
- raise ValueError("Vocabulary file %s not found.", vocabulary_path)
 
 
 def sentence_to_token_ids(sentence, vocabulary, tokenizer=None, normalize_digits=True):
-
-if tokenizer:
+ if not tokenizer:
+tokenizer = basic_tokenizer
  words = tokenizer(sentence)
- else:
- words = basic_tokenizer(sentence)
- if not normalize_digits:
- return [vocabulary.get(w, UNK_ID) for w in words]
- # Normalize digits by 0 before looking words up in the vocabulary.
- return [vocabulary.get(re.sub(_DIGIT_RE, b"0", w), UNK_ID) for w in words]
+ if not normalize_digits:
+ return [vocabulary.get(w, UNK_ID) for w in words]
+ # Normalize digits by 0 before looking words up in the vocabulary.
+ return [vocabulary.get(re.sub(_DIGIT_RE, '0', w), UNK_ID) for w in words]
 
 
 def data_to_token_ids(data_path, target_path, vocabulary_path,
  tokenizer=None, normalize_digits=True):
-
- if not gfile.Exists(target_path):
- print("Tokenizing data in %s" % data_path)
- vocab, _ = initialize_vocabulary(vocabulary_path)
- with gfile.GFile(data_path, mode="rb") as data_file:
- with gfile.GFile(target_path, mode="w") as tokens_file:
- counter = 0
- for line in data_file:
- counter += 1
- if counter % 100000 == 0:
- print(" tokenizing line %d" % counter)
- token_ids = sentence_to_token_ids(line, vocab, tokenizer,
- normalize_digits)
- tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
-
-
-
-def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size, dec_vocabulary_size, tokenizer=None):
-
+ if not gfile.Exists(target_path):
+ print("Tokenizing data in %s" % data_path)
+ vocab, _ = initialize_vocabulary(vocabulary_path)
+ with gfile.GFile(data_path, mode="rb") as data_file:
+ with gfile.GFile(target_path, mode="w") as tokens_file:
+ for counter, line in enumerate(data_file, 1):
+ if counter % 100000 == 0:
+ print(" tokenizing line %d" % counter)
+ token_ids = sentence_to_token_ids(line, vocab, tokenizer,
+ normalize_digits)
+ tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
+
+
+def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_dec, enc_vocabulary_size,
+ dec_vocabulary_size, tokenizer=None):
  # Create vocabularies of the appropriate sizes.
  enc_vocab_path = os.path.join(working_directory, "vocab%d.enc" % enc_vocabulary_size)
  dec_vocab_path = os.path.join(working_directory, "vocab%d.dec" % dec_vocabulary_size)
@@ -143,4 +147,4 @@ def prepare_custom_data(working_directory, train_enc, train_dec, test_enc, test_
  data_to_token_ids(test_enc, enc_dev_ids_path, enc_vocab_path, tokenizer)
  data_to_token_ids(test_dec, dec_dev_ids_path, dec_vocab_path, tokenizer)
 
- return (enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path)
+ return enc_train_ids_path, dec_train_ids_path, enc_dev_ids_path, dec_dev_ids_path, enc_vocab_path, dec_vocab_path