From cf3b65002dcbd638383f05dc3f6a2d51aa1bb9e3 Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Tue, 30 May 2017 19:13:52 +0000
Subject: [PATCH 01/11] Initial Visual Genome commit

---
 chainercv/datasets/__init__.py                |   2 +
 chainercv/datasets/visual_genome/__init__.py  |   0
 ...sual_genome_region_descriptions_dataset.py | 266 ++++++++++++++++++
 .../visual_genome/visual_genome_utils.py      |  91 ++++++
 4 files changed, 359 insertions(+)
 create mode 100644 chainercv/datasets/visual_genome/__init__.py
 create mode 100644 chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
 create mode 100644 chainercv/datasets/visual_genome/visual_genome_utils.py

diff --git a/chainercv/datasets/__init__.py b/chainercv/datasets/__init__.py
index d1088a6f00..6ff38e22c1 100644
--- a/chainercv/datasets/__init__.py
+++ b/chainercv/datasets/__init__.py
@@ -6,6 +6,8 @@
 from chainercv.datasets.cub.cub_label_dataset import CUBLabelDataset  # NOQA
 from chainercv.datasets.online_products.online_products_dataset import OnlineProductsDataset  # NOQA
 from chainercv.datasets.transform_dataset import TransformDataset  # NOQA
+from chainercv.datasets.visual_genome.visual_genome_utils import VisualGenomeDatasetBase  # NQDA
+from chainercv.datasets.visual_genome.visual_genome_region_descriptions_dataset import VisualGenomeRegionDescriptionsDataset  # NQDA
 from chainercv.datasets.voc.voc_detection_dataset import VOCDetectionDataset  # NOQA
 from chainercv.datasets.voc.voc_semantic_segmentation_dataset import VOCSemanticSegmentationDataset  # NOQA
 from chainercv.datasets.voc.voc_utils import voc_detection_label_names  # NOQA
diff --git a/chainercv/datasets/visual_genome/__init__.py b/chainercv/datasets/visual_genome/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
new file mode 100644
index 0000000000..437c9f9f9d
--- /dev/null
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -0,0 +1,266 @@
+from collections import defaultdict
+from collections import Counter
+import json
+import os
+import pickle
+import six
+import string
+
+import chainer
+from chainer.dataset import download
+import numpy as np
+
+from chainercv.datasets.visual_genome.visual_genome_utils import root
+from chainercv.datasets.visual_genome.visual_genome_utils import \
+    get_extract_data
+from chainercv.datasets.visual_genome.visual_genome_utils import \
+    get_region_descriptions
+from chainercv.datasets.visual_genome.visual_genome_utils import \
+    VisualGenomeDatasetBase
+from chainercv.transforms import resize
+from chainercv.transforms import resize_bbox
+
+
+def get_region_ids(region_descriptions_path):
+
+    """Returns a dict that maps image ids to region ids.
+
+    For each image id, there is a list of region ids.
+
+    """
+    data_root = download.get_dataset_directory(root)
+    base_path = os.path.join(data_root, 'region_ids.pkl')
+
+    def creator(path):
+        print('Caching region IDs')
+        region_ids = defaultdict(list)
+        with open(region_descriptions_path) as f:
+            region_descriptions = json.load(f)
+            for region_description in region_descriptions:
+                for region in region_description['regions']:
+                    img_id = region['image_id']
+                    region_id = region['region_id']
+                    region_ids[img_id].append(region_id)
+        pickle.dump(dict(region_ids), open(base_path, 'wb'))
+        return region_ids
+
+    def loader(path):
+        return pickle.load(open(base_path, 'rb'))
+
+    return download.cache_or_load_file(base_path, creator, loader)
+
+
+def get_regions(region_descriptions_path):
+
+    """Returns a dict that maps region ids to a bbox tuples.
+
+    Each bbox tuple is defined as (x_min, y_min, x_max, y_max).
+
+    """
+    data_root = download.get_dataset_directory(root)
+    base_path = os.path.join(data_root, 'regions.pkl')
+
+    def creator(path):
+        print('Caching region bounding boxes...')
+        regions = {}
+        with open(region_descriptions_path) as f:
+            region_descriptions = json.load(f)
+            for region_description in region_descriptions:
+                for region in region_description['regions']:
+                    region_id = r['region_id']  # int
+                    xmin = region['x']
+                    ymin = region['y']
+                    xmax = xmin + region['width']
+                    ymax = ymin + region['height']
+                    regions[region_id] = (xmin, ymin, xmax, ymax)
+        pickle.dump(regions, open(base_path, 'wb'))
+        return regions
+
+    def loader(path):
+        return pickle.load(open(base_path, 'rb'))
+
+    return download.cache_or_load_file(base_path, creator, loader)
+
+
+def get_phrases(region_descriptions_path, min_token_instances):
+
+    """Return a dict that maps a region_id to its corresponding phrase.
+
+    Phrases in this case are represented as a list of ints of word_ids
+
+    """
+    data_root = download.get_dataset_directory(root)
+    base_path = os.path.join(data_root,
+                             'phrases_{}.pkl'.format(min_token_instances))
+
+    def creator(path):
+        print('Caching region phrases...')
+        phrases = {}
+        vocab = retrieve_word_vocabulary(region_descriptions_path,
+                                         min_token_instances)
+        with open(region_descriptions_path) as f:
+            region_descriptions = json.load(f)
+            for region_description in region_descriptions:
+                for region in region_description['regions']:
+                    region_id = region['region_id']
+                    word_ids = []
+                    for word in preprocess_phrase(region['phrase']).split():
+                        if word not in vocab:
+                            word = '<unk>'
+                        word_id = vocab[word]
+                        word_ids.append(word_id)
+                    phrases[region_id] = word_ids
+        pickle.dump(phrases, open(base_path, 'wb'))
+        return phrases
+
+    def loader(path):
+        return pickle.load(open(base_path, 'rb'))
+
+    return download.cache_or_load_file(base_path, creator, loader)
+
+
+def get_vocabulary(region_descriptions_path='auto', min_token_instances=15):
+
+    """Creates a vocabulary based on the region descriptions of Visual Genome.
+
+    A vocabulary is a dictionary that maps each word (str) to its
+    corresponding id (int). Rare words are treated as unknown words, i.e.
+    <unk> and are excluded for the dictionary.
+
+    Args:
+        min_token_instances (int): When words appear less than this times, they
+            will be treated as <unk>.
+
+    Returns:
+        dict: A dictionary mapping words to their corresponding ids.
+
+    """
+    if region_descriptions_path == 'auto':
+        region_descriptions_path = get_region_descriptions()
+    return retrieve_word_vocabulary(region_descriptions_path,
+                                    min_token_instances)
+
+
+def retrieve_word_vocabulary(region_descriptions_path, min_token_instances):
+    def creator(path):
+        print('Creating vocabulary (ignoring words that appear less than '
+              '{} times)...'.format(min_token_instances))
+        words = load_words(region_descriptions_path,
+                           min_token_instances=min_token_instances, sort=True)
+        vocab = {}  # word (str) -> word_id (int)
+        index = 0
+        with open(path, 'w') as f:
+            for word in words:
+                if word not in vocab:
+                    vocab[word] = index
+                    index += 1
+                    f.write(word + '\n')
+        return vocab
+
+    def loader(path):
+        vocab = {}
+        with open(path) as f:
+            for i, word in enumerate(f):
+                vocab[word.strip()] = i
+        return vocab
+
+    data_root = download.get_dataset_directory(root)
+    base_path = os.path.join(data_root,
+                             'vocab_{}.txt'.format(min_token_instances))
+    return download.cache_or_load_file(base_path, creator, loader)
+
+
+def load_words(region_descriptions_path, min_token_instances=None, sort=True):
+    word_counts = Counter()
+    with open(region_descriptions_path) as f:
+        region_descriptions = json.load(f)
+        for region_description in region_descriptions:
+            for region in region_description['regions']:
+                for word in preprocess_phrase(region['phrase']).split():
+                    word_counts[word] += 1
+
+    words = ['<unk>', '<eos>']
+    for word, count in six.iteritems(word_counts):
+        if min_token_instances is None or count >= min_token_instances:
+            words.append(word)
+
+    if sort:
+        words = sorted(words)
+
+    return words
+
+
+def preprocess_phrase(phrase):
+    """Preprocess a phrase similar to the DenseCap implementation.
+
+    Certain non-ascii characters are replaced, punctuations are removed and all
+    characeters are lower-cased.
+
+    Please refer to the following implementation.
+    https://github.com/jcjohnson/densecap/blob/master/preprocess.py
+
+    Args:
+        phrse (str): A phrase to process.
+
+    Returns:
+        str: A processed phrase.
+    """
+    replacements = {
+        u'\xa2': u'cent',
+        u'\xb0': u' degree',
+        u'\xbd': u'half',
+        u'\xe7': u'c',
+        u'\xe8': u'e',
+        u'\xe9': u'e',
+        u'\xfb': u'u',
+        u'\u2014': u'-',
+        u'\u2026': u'',
+        u'\u2122': u'',
+    }
+
+    for k, v in six.iteritems(replacements):
+        phrase = phrase.replace(k, v)
+    trans = str.maketrans('', '', string.punctuation)
+    return str(phrase).lower().translate(trans)
+
+
+
+class VisualGenomeRegionDescriptionsDataset(VisualGenomeDatasetBase):
+
+    """Region description class for Visual Genome dataset.
+
+    """
+
+    def __init__(self, data_dir='auto', image_data='auto',
+                 region_descriptions='auto', min_token_instances=15,
+                 img_size=(720, 720)):
+        super(VisualGenomeRegionDescriptionsDataset, self).__init__(
+            data_dir=data_dir, image_data=image_data)
+
+        if region_descriptions == 'auto':
+            region_descriptions = get_region_descriptions()
+
+        self.region_ids = get_region_ids(region_descriptions)
+        self.regions = get_regions(region_descriptions)
+        self.phrases = get_phrases(region_descriptions,
+                                   min_token_instances=min_token_instances)
+        self.img_size = img_size
+
+    def get_example(self, i):
+        img_id = self.get_image_id(i)
+        img = self.get_image(img_id)
+
+        regions = []
+        phrases = []
+        for region_id in self.region_ids[img_id]:
+            regions.append(self.regions[region_id])
+            phrases.append(self.phrases[region_id])
+        regions = np.vstack(regions)
+
+        if self.img_size is not None:
+            h_orig, w_orig = img.shape[1:]
+            img = resize(img, self.img_size)
+            h, w = img.shape[1:]
+            regions = resize_bbox(regions, (w_orig, h_orig), (w, h))
+
+        return img, regions, phrases
diff --git a/chainercv/datasets/visual_genome/visual_genome_utils.py b/chainercv/datasets/visual_genome/visual_genome_utils.py
new file mode 100644
index 0000000000..b233d980db
--- /dev/null
+++ b/chainercv/datasets/visual_genome/visual_genome_utils.py
@@ -0,0 +1,91 @@
+import json
+import os
+
+import chainer
+from chainer.dataset import download
+
+from chainercv import utils
+
+root = 'pfnet/chainer/visual_genome'
+
+vg_100k_url = 'https://cs.stanford.edu/people/rak248/VG_100K_2/images.zip'
+vg_100k_2_url = 'https://cs.stanford.edu/people/rak248/VG_100K_2/images2.zip'
+image_data_url = 'http://visualgenome.org/static/data/dataset/' \
+    'image_data.json.zip'
+region_descriptions_url = 'http://visualgenome.org/static/data/dataset/' \
+    'region_descriptions.json.zip'
+
+
+def get_visual_genome():
+
+    def move_files(src_dir, dst_dir):
+        for f in os.listdir(src_dir):
+            src = os.path.join(src_dir, f)
+            if os.path.isfile(src):
+                dst = os.path.join(dst_dir, f)
+                os.rename(src, dst)
+        os.rmdir(src_dir)
+
+    data_root = download.get_dataset_directory(root)
+    base_path = os.path.join(data_root, 'VG_100K_ALL')
+    if os.path.exists(base_path):
+        return base_path
+
+    os.mkdir(base_path)
+    move_files(get_extract_data(vg_100k_url, data_root, 'VG_100K'), base_path)
+    move_files(get_extract_data(vg_100k_2_url, data_root, 'VG_100K_2'),
+               base_path)
+
+    return base_path
+
+
+def get_image_data():
+    data_root = download.get_dataset_directory(root)
+    return get_extract_data(image_data_url, data_root, 'image_data.json')
+
+
+def get_region_descriptions():
+    data_root = download.get_dataset_directory(root)
+    return get_extract_data(region_descriptions_url, data_root,
+                            'region_description.json')
+
+
+def get_extract_data(url, data_root, member_path):
+    base_path = os.path.join(data_root, member_path)
+
+    if os.path.exists(base_path):
+        return base_path
+
+    download_file_path = utils.cached_download(url)
+    ext = os.path.splitext(url)[1]
+    utils.extractall(download_file_path, data_root, ext)
+
+    return base_path
+
+
+class VisualGenomeDatasetBase(chainer.dataset.DatasetMixin):
+
+    """Base class for Visual Genome dataset.
+
+    """
+
+    def __init__(self, data_dir='auto', image_data='auto'):
+        if data_dir == 'auto':
+            data_dir = get_visual_genome()
+        if image_data == 'auto':
+            image_data = get_image_data()
+        self.data_dir = data_dir
+
+        with open(image_data, 'r') as f:
+            img_ids = [img_data['image_id'] for img_data in json.load(f)]
+        self.img_ids = sorted(img_ids)
+
+    def __len__(self):
+        return len(self.img_ids)
+
+    def get_image_id(self, i):
+        return self.img_ids[i]
+
+    def get_image(self, img_id):
+        img_path = os.path.join(self.data_dir, str(img_id) + '.jpg')
+        return utils.read_image(img_path, color=True)

From 3e8ff003383f54a399cf941a4a9cef986349bcb8 Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Tue, 30 May 2017 20:30:52 +0000
Subject: [PATCH 02/11] Add docs and refactor functions

---
 ...sual_genome_region_descriptions_dataset.py | 218 +++++++++---------
 .../visual_genome/visual_genome_utils.py      |  54 +++--
 2 files changed, 142 insertions(+), 130 deletions(-)

diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
index 437c9f9f9d..b6f807baac 100644
--- a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -1,38 +1,96 @@
-from collections import defaultdict
 from collections import Counter
+from collections import defaultdict
 import json
 import os
 import pickle
 import six
 import string
 
-import chainer
 from chainer.dataset import download
 import numpy as np
 
-from chainercv.datasets.visual_genome.visual_genome_utils import root
-from chainercv.datasets.visual_genome.visual_genome_utils import \
-    get_extract_data
 from chainercv.datasets.visual_genome.visual_genome_utils import \
     get_region_descriptions
+from chainercv.datasets.visual_genome.visual_genome_utils import root
 from chainercv.datasets.visual_genome.visual_genome_utils import \
     VisualGenomeDatasetBase
 from chainercv.transforms import resize
 from chainercv.transforms import resize_bbox
 
 
-def get_region_ids(region_descriptions_path):
+def get_vocabulary(region_descriptions='auto', min_token_instances=15):
+
+    """Creates a vocabulary based on the region descriptions of Visual Genome.
+
+    A vocabulary is a dictionary that maps each word (str) to its
+    corresponding id (int). Rare words are treated as unknown words, i.e.
+    <unk> and are excluded for the dictionary.
+
+    Args:
+        min_token_instances (int): When words appear less than this times, they
+            will be treated as <unk>.
+
+    Returns:
+        dict: A dictionary mapping words to their corresponding ids.
+
+    """
+    if region_descriptions == 'auto':
+        region_descriptions = get_region_descriptions()
+    return _create_word_vocabulary(region_descriptions, min_token_instances)
+
+
+class VisualGenomeRegionDescriptionsDataset(VisualGenomeDatasetBase):
+
+    """Region description class for Visual Genome dataset.
+
+    """
+
+    def __init__(self, data_dir='auto', image_data='auto',
+                 region_descriptions='auto', min_token_instances=15,
+                 img_size=(720, 720)):
+        super(VisualGenomeRegionDescriptionsDataset, self).__init__(
+            data_dir=data_dir, image_data=image_data)
+
+        if region_descriptions == 'auto':
+            region_descriptions = get_region_descriptions()
+
+        self.img_size = img_size
+
+        self.region_ids = _get_region_ids(region_descriptions)
+        self.regions = _get_regions(region_descriptions)
+        self.phrases = _get_phrases(region_descriptions,
+                                    min_token_instances=min_token_instances)
+
+    def get_example(self, i):
+        img_id = self.get_image_id(i)
+        img = self.get_image(img_id)
+
+        regions = []
+        phrases = []
+        for region_id in self.region_ids[img_id]:
+            regions.append(self.regions[region_id])
+            phrases.append(self.phrases[region_id])
+        regions = np.vstack(regions)
+
+        if self.img_size is not None:
+            h_orig, w_orig = img.shape[1:]
+            img = resize(img, self.img_size)
+            h, w = img.shape[1:]
+            regions = resize_bbox(regions, (w_orig, h_orig), (w, h))
+
+        return img, regions, phrases
+
 
-    """Returns a dict that maps image ids to region ids.
+def _get_region_ids(region_descriptions_path):
 
-    For each image id, there is a list of region ids.
+    """Image ID (int) -> Region IDs (list of int).
 
     """
     data_root = download.get_dataset_directory(root)
     base_path = os.path.join(data_root, 'region_ids.pkl')
 
     def creator(path):
-        print('Caching region IDs')
+        print('Caching Visual Genome region IDs...')
         region_ids = defaultdict(list)
         with open(region_descriptions_path) as f:
             region_descriptions = json.load(f)
@@ -50,24 +108,22 @@ def loader(path):
     return download.cache_or_load_file(base_path, creator, loader)
 
 
-def get_regions(region_descriptions_path):
-
-    """Returns a dict that maps region ids to a bbox tuples.
+def _get_regions(region_descriptions_path):
 
-    Each bbox tuple is defined as (x_min, y_min, x_max, y_max).
+    """Region ID (int) -> Region bounding box (tuple of int).
 
     """
     data_root = download.get_dataset_directory(root)
     base_path = os.path.join(data_root, 'regions.pkl')
 
     def creator(path):
-        print('Caching region bounding boxes...')
+        print('Caching Visual Genome region bounding boxes...')
         regions = {}
         with open(region_descriptions_path) as f:
             region_descriptions = json.load(f)
             for region_description in region_descriptions:
                 for region in region_description['regions']:
-                    region_id = r['region_id']  # int
+                    region_id = region['region_id']  # int
                     xmin = region['x']
                     ymin = region['y']
                     xmax = xmin + region['width']
@@ -82,11 +138,9 @@ def loader(path):
     return download.cache_or_load_file(base_path, creator, loader)
 
 
-def get_phrases(region_descriptions_path, min_token_instances):
+def _get_phrases(region_descriptions_path, min_token_instances):
 
-    """Return a dict that maps a region_id to its corresponding phrase.
-
-    Phrases in this case are represented as a list of ints of word_ids
+    """Region ID (int) -> Phrase (list of int).
 
     """
     data_root = download.get_dataset_directory(root)
@@ -94,22 +148,22 @@ def get_phrases(region_descriptions_path, min_token_instances):
                              'phrases_{}.pkl'.format(min_token_instances))
 
     def creator(path):
-        print('Caching region phrases...')
+        print('Caching Visual Genome region descriptions...')
         phrases = {}
-        vocab = retrieve_word_vocabulary(region_descriptions_path,
-                                         min_token_instances)
+        vocab = _create_word_vocabulary(region_descriptions_path,
+                                        min_token_instances)
         with open(region_descriptions_path) as f:
             region_descriptions = json.load(f)
             for region_description in region_descriptions:
                 for region in region_description['regions']:
                     region_id = region['region_id']
-                    word_ids = []
-                    for word in preprocess_phrase(region['phrase']).split():
+                    phrase = []
+                    for word in _preprocess_phrase(region['phrase']).split():
                         if word not in vocab:
                             word = '<unk>'
                         word_id = vocab[word]
-                        word_ids.append(word_id)
-                    phrases[region_id] = word_ids
+                        phrase.append(word_id)
+                    phrases[region_id] = phrase
         pickle.dump(phrases, open(base_path, 'wb'))
         return phrases
 
@@ -119,35 +173,21 @@ def loader(path):
     return download.cache_or_load_file(base_path, creator, loader)
 
 
-def get_vocabulary(region_descriptions_path='auto', min_token_instances=15):
+def _create_word_vocabulary(region_descriptions_path, min_token_instances):
 
-    """Creates a vocabulary based on the region descriptions of Visual Genome.
-
-    A vocabulary is a dictionary that maps each word (str) to its
-    corresponding id (int). Rare words are treated as unknown words, i.e.
-    <unk> and are excluded for the dictionary.
-
-    Args:
-        min_token_instances (int): When words appear less than this times, they
-            will be treated as <unk>.
-
-    Returns:
-        dict: A dictionary mapping words to their corresponding ids.
+    """Word (str) -> Word ID (int).
 
     """
-    if region_descriptions_path == 'auto':
-        region_descriptions_path = get_region_descriptions()
-    return retrieve_word_vocabulary(region_descriptions_path,
-                                    min_token_instances)
-
+    data_root = download.get_dataset_directory(root)
+    base_path = os.path.join(data_root,
+                             'vocab_{}.txt'.format(min_token_instances))
 
-def retrieve_word_vocabulary(region_descriptions_path, min_token_instances):
     def creator(path):
-        print('Creating vocabulary (ignoring words that appear less than '
-              '{} times)...'.format(min_token_instances))
-        words = load_words(region_descriptions_path,
-                           min_token_instances=min_token_instances, sort=True)
-        vocab = {}  # word (str) -> word_id (int)
+        print('Creating vocabulary from region descriptions (ignoring words '
+              'that appear less than {} times)...'.format(min_token_instances))
+        words = _load_words(region_descriptions_path,
+                            min_token_instances=min_token_instances)
+        vocab = {}
         index = 0
         with open(path, 'w') as f:
             for word in words:
@@ -164,47 +204,36 @@ def loader(path):
                 vocab[word.strip()] = i
         return vocab
 
-    data_root = download.get_dataset_directory(root)
-    base_path = os.path.join(data_root,
-                             'vocab_{}.txt'.format(min_token_instances))
     return download.cache_or_load_file(base_path, creator, loader)
 
 
-def load_words(region_descriptions_path, min_token_instances=None, sort=True):
+def _load_words(region_descriptions_path, min_token_instances):
+    # Count the number of occurrences for each word in all region descriptions
+    # to only include those words that appear at least a few times
     word_counts = Counter()
     with open(region_descriptions_path) as f:
         region_descriptions = json.load(f)
         for region_description in region_descriptions:
             for region in region_description['regions']:
-                for word in preprocess_phrase(region['phrase']).split():
+                for word in _preprocess_phrase(region['phrase']).split():
                     word_counts[word] += 1
 
-    words = ['<unk>', '<eos>']
+    words = []
     for word, count in six.iteritems(word_counts):
         if min_token_instances is None or count >= min_token_instances:
             words.append(word)
+    words.append('<unk>')
+    words.append('<eos>')
 
-    if sort:
-        words = sorted(words)
+    # Sort to make sure that word orders are consistent
+    words = sorted(words)
 
     return words
 
 
-def preprocess_phrase(phrase):
-    """Preprocess a phrase similar to the DenseCap implementation.
-
-    Certain non-ascii characters are replaced, punctuations are removed and all
-    characeters are lower-cased.
-
-    Please refer to the following implementation.
-    https://github.com/jcjohnson/densecap/blob/master/preprocess.py
-
-    Args:
-        phrse (str): A phrase to process.
-
-    Returns:
-        str: A processed phrase.
-    """
+def _preprocess_phrase(phrase):
+    # Preprocess phrases according to the DenseCap implementation
+    # https://github.com/jcjohnson/densecap/blob/master/preprocess.py
     replacements = {
         u'\xa2': u'cent',
         u'\xb0': u' degree',
@@ -220,47 +249,6 @@ def preprocess_phrase(phrase):
 
     for k, v in six.iteritems(replacements):
         phrase = phrase.replace(k, v)
+
     trans = str.maketrans('', '', string.punctuation)
     return str(phrase).lower().translate(trans)
-
-
-
-class VisualGenomeRegionDescriptionsDataset(VisualGenomeDatasetBase):
-
-    """Region description class for Visual Genome dataset.
-
-    """
-
-    def __init__(self, data_dir='auto', image_data='auto',
-                 region_descriptions='auto', min_token_instances=15,
-                 img_size=(720, 720)):
-        super(VisualGenomeRegionDescriptionsDataset, self).__init__(
-            data_dir=data_dir, image_data=image_data)
-
-        if region_descriptions == 'auto':
-            region_descriptions = get_region_descriptions()
-
-        self.region_ids = get_region_ids(region_descriptions)
-        self.regions = get_regions(region_descriptions)
-        self.phrases = get_phrases(region_descriptions,
-                                   min_token_instances=min_token_instances)
-        self.img_size = img_size
-
-    def get_example(self, i):
-        img_id = self.get_image_id(i)
-        img = self.get_image(img_id)
-
-        regions = []
-        phrases = []
-        for region_id in self.region_ids[img_id]:
-            regions.append(self.regions[region_id])
-            phrases.append(self.phrases[region_id])
-        regions = np.vstack(regions)
-
-        if self.img_size is not None:
-            h_orig, w_orig = img.shape[1:]
-            img = resize(img, self.img_size)
-            h, w = img.shape[1:]
-            regions = resize_bbox(regions, (w_orig, h_orig), (w, h))
-
-        return img, regions, phrases
diff --git a/chainercv/datasets/visual_genome/visual_genome_utils.py b/chainercv/datasets/visual_genome/visual_genome_utils.py
index b233d980db..3b345f9299 100644
--- a/chainercv/datasets/visual_genome/visual_genome_utils.py
+++ b/chainercv/datasets/visual_genome/visual_genome_utils.py
@@ -18,7 +18,15 @@
 
 def get_visual_genome():
 
+    """Get the default path to the Visual Genome image directory.
+
+    Returns:
+        str: A path to the image directory.
+
+    """
+
     def move_files(src_dir, dst_dir):
+        # Move all files in the src_dir to the dst_dir and remove the src_dir
         for f in os.listdir(src_dir):
             src = os.path.join(src_dir, f)
             if os.path.isfile(src):
@@ -28,39 +36,42 @@ def move_files(src_dir, dst_dir):
 
     data_root = download.get_dataset_directory(root)
     base_path = os.path.join(data_root, 'VG_100K_ALL')
+
     if os.path.exists(base_path):
         return base_path
 
+    print('Caching Visual Genome image files...')
     os.mkdir(base_path)
-    move_files(get_extract_data(vg_100k_url, data_root, 'VG_100K'), base_path)
-    move_files(get_extract_data(vg_100k_2_url, data_root, 'VG_100K_2'),
+    move_files(_get_extract_data(vg_100k_url, data_root, 'VG_100K'), base_path)
+    move_files(_get_extract_data(vg_100k_2_url, data_root, 'VG_100K_2'),
                base_path)
 
     return base_path
 
 
 def get_image_data():
-    data_root = download.get_dataset_directory(root)
-    return get_extract_data(image_data_url, data_root, 'image_data.json')
 
+    """Get the default path to the image data JSON file.
 
-def get_region_descriptions():
+    Returns:
+        str: A path to the image data JSON file.
+
+    """
     data_root = download.get_dataset_directory(root)
-    return get_extract_data(region_descriptions_url, data_root,
-                            'region_description.json')
+    return _get_extract_data(image_data_url, data_root, 'image_data.json')
 
 
-def get_extract_data(url, data_root, member_path):
-    base_path = os.path.join(data_root, member_path)
+def get_region_descriptions():
 
-    if os.path.exists(base_path):
-        return base_path
+    """Get the default path to the region descriptions JSON file.
 
-    download_file_path = utils.cached_download(url)
-    ext = os.path.splitext(url)[1]
-    utils.extractall(download_file_path, data_root, ext)
+    Returns:
+        str: A path to the region descriptions JSON file.
 
-    return base_path
+    """
+    data_root = download.get_dataset_directory(root)
+    return _get_extract_data(region_descriptions_url, data_root,
+                             'region_descriptions.json')
 
 
 class VisualGenomeDatasetBase(chainer.dataset.DatasetMixin):
@@ -89,3 +100,16 @@ def get_image_id(self, i):
     def get_image(self, img_id):
         img_path = os.path.join(self.data_dir, str(img_id) + '.jpg')
         return utils.read_image(img_path, color=True)
+
+
+def _get_extract_data(url, data_root, member_path):
+    base_path = os.path.join(data_root, member_path)
+
+    if os.path.exists(base_path):
+        return base_path
+
+    download_file_path = utils.cached_download(url)
+    ext = os.path.splitext(url)[1]
+    utils.extractall(download_file_path, data_root, ext)
+
+    return base_path

From 0820e0605f094972858efb2b748434c157ba9882 Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Tue, 30 May 2017 20:33:18 +0000
Subject: [PATCH 03/11] Minor cosmetic

---
 .../visual_genome/visual_genome_region_descriptions_dataset.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
index b6f807baac..864375c398 100644
--- a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -58,8 +58,7 @@ def __init__(self, data_dir='auto', image_data='auto',
 
         self.region_ids = _get_region_ids(region_descriptions)
         self.regions = _get_regions(region_descriptions)
-        self.phrases = _get_phrases(region_descriptions,
-                                    min_token_instances=min_token_instances)
+        self.phrases = _get_phrases(region_descriptions, min_token_instances)
 
     def get_example(self, i):
         img_id = self.get_image_id(i)

From 64f01d9142d4092963fab1e104b2dff26d8a3d6a Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Thu, 1 Jun 2017 05:52:06 +0000
Subject: [PATCH 04/11] Cap phrases to given length and pad with 0

---
 ...sual_genome_region_descriptions_dataset.py | 38 ++++++++++++-------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
index 864375c398..ccb1f8e778 100644
--- a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -47,7 +47,7 @@ class VisualGenomeRegionDescriptionsDataset(VisualGenomeDatasetBase):
 
     def __init__(self, data_dir='auto', image_data='auto',
                  region_descriptions='auto', min_token_instances=15,
-                 img_size=(720, 720)):
+                 max_token_length=15, img_size=(720, 720)):
         super(VisualGenomeRegionDescriptionsDataset, self).__init__(
             data_dir=data_dir, image_data=image_data)
 
@@ -58,7 +58,8 @@ def __init__(self, data_dir='auto', image_data='auto',
 
         self.region_ids = _get_region_ids(region_descriptions)
         self.regions = _get_regions(region_descriptions)
-        self.phrases = _get_phrases(region_descriptions, min_token_instances)
+        self.phrases = _get_phrases(region_descriptions, min_token_instances,
+                                    max_token_length)
 
     def get_example(self, i):
         img_id = self.get_image_id(i)
@@ -67,9 +68,13 @@ def get_example(self, i):
         regions = []
         phrases = []
         for region_id in self.region_ids[img_id]:
-            regions.append(self.regions[region_id])
-            phrases.append(self.phrases[region_id])
-        regions = np.vstack(regions)
+            phrase = self.phrases[region_id]
+            if phrase is not None:  # If phrase wasn't too long and exlcluded
+                regions.append(self.regions[region_id])
+                phrases.append(self.phrases[region_id])
+        regions = np.vstack(regions).astype(np.float32)
+        phrases = np.vstack(phrases).astype(np.int32)
+
 
         if self.img_size is not None:
             h_orig, w_orig = img.shape[1:]
@@ -137,7 +142,8 @@ def loader(path):
     return download.cache_or_load_file(base_path, creator, loader)
 
 
-def _get_phrases(region_descriptions_path, min_token_instances):
+def _get_phrases(region_descriptions_path, min_token_instances,
+                 max_token_length):
 
     """Region ID (int) -> Phrase (list of int).
 
@@ -156,13 +162,19 @@ def creator(path):
             for region_description in region_descriptions:
                 for region in region_description['regions']:
                     region_id = region['region_id']
-                    phrase = []
-                    for word in _preprocess_phrase(region['phrase']).split():
-                        if word not in vocab:
-                            word = '<unk>'
-                        word_id = vocab[word]
-                        phrase.append(word_id)
-                    phrases[region_id] = phrase
+                    tokens = _preprocess_phrase(region['phrase']).split()
+                    if max_token_length > 0 and \
+                            len(tokens) <= max_token_length:
+                        phrase = np.zeros(max_token_length, dtype=np.int32)
+                        for i, token in enumerate(tokens):
+                            if token not in vocab:
+                                token = '<unk>'
+                            token_id = vocab[token]
+                            phrase[i] = token_id
+                        phrases[region_id] = phrase
+                    else:
+                        phrases[region_id] = None
+
         pickle.dump(phrases, open(base_path, 'wb'))
         return phrases
 

From cc55cdb3cf2678ef7b2bb293c1d5d27227f9a704 Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Thu, 1 Jun 2017 06:00:13 +0000
Subject: [PATCH 05/11] Fix PEP8

---
 .../visual_genome/visual_genome_region_descriptions_dataset.py   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
index ccb1f8e778..14d0da609b 100644
--- a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -75,7 +75,6 @@ def get_example(self, i):
         regions = np.vstack(regions).astype(np.float32)
         phrases = np.vstack(phrases).astype(np.int32)
 
-
         if self.img_size is not None:
             h_orig, w_orig = img.shape[1:]
             img = resize(img, self.img_size)

From 82ca3c82c6e7b99b9c0864c5ece83c480b450347 Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Thu, 1 Jun 2017 06:14:15 +0000
Subject: [PATCH 06/11] Exclude long captions completely

---
 .../visual_genome_region_descriptions_dataset.py           | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
index 14d0da609b..64abdeb329 100644
--- a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -68,8 +68,9 @@ def get_example(self, i):
         regions = []
         phrases = []
         for region_id in self.region_ids[img_id]:
-            phrase = self.phrases[region_id]
-            if phrase is not None:  # If phrase wasn't too long and exlcluded
+            # Phrases that are too long are excluded in the preprocessing,
+            # so only include regions with actual phrases
+            if region_id in self.phrases:
                 regions.append(self.regions[region_id])
                 phrases.append(self.phrases[region_id])
         regions = np.vstack(regions).astype(np.float32)
@@ -171,8 +172,6 @@ def creator(path):
                             token_id = vocab[token]
                             phrase[i] = token_id
                         phrases[region_id] = phrase
-                    else:
-                        phrases[region_id] = None
 
         pickle.dump(phrases, open(base_path, 'wb'))
         return phrases

From a9deca0d31b6e492df9d30af744b6859f8ce7944 Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Thu, 1 Jun 2017 07:05:18 +0000
Subject: [PATCH 07/11] Fix PEP8

---
 .../visual_genome_region_descriptions_dataset.py            | 6 ------
 chainercv/datasets/visual_genome/visual_genome_utils.py     | 5 -----
 2 files changed, 11 deletions(-)

diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
index 64abdeb329..48a40e9d0a 100644
--- a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -19,7 +19,6 @@
 
 
 def get_vocabulary(region_descriptions='auto', min_token_instances=15):
-
     """Creates a vocabulary based on the region descriptions of Visual Genome.
 
     A vocabulary is a dictionary that maps each word (str) to its
@@ -40,7 +39,6 @@ def get_vocabulary(region_descriptions='auto', min_token_instances=15):
 
 
 class VisualGenomeRegionDescriptionsDataset(VisualGenomeDatasetBase):
-
     """Region description class for Visual Genome dataset.
 
     """
@@ -86,7 +84,6 @@ def get_example(self, i):
 
 
 def _get_region_ids(region_descriptions_path):
-
     """Image ID (int) -> Region IDs (list of int).
 
     """
@@ -113,7 +110,6 @@ def loader(path):
 
 
 def _get_regions(region_descriptions_path):
-
     """Region ID (int) -> Region bounding box (tuple of int).
 
     """
@@ -144,7 +140,6 @@ def loader(path):
 
 def _get_phrases(region_descriptions_path, min_token_instances,
                  max_token_length):
-
     """Region ID (int) -> Phrase (list of int).
 
     """
@@ -183,7 +178,6 @@ def loader(path):
 
 
 def _create_word_vocabulary(region_descriptions_path, min_token_instances):
-
     """Word (str) -> Word ID (int).
 
     """
diff --git a/chainercv/datasets/visual_genome/visual_genome_utils.py b/chainercv/datasets/visual_genome/visual_genome_utils.py
index 3b345f9299..0c39cabcad 100644
--- a/chainercv/datasets/visual_genome/visual_genome_utils.py
+++ b/chainercv/datasets/visual_genome/visual_genome_utils.py
@@ -17,14 +17,12 @@
 
 
 def get_visual_genome():
-
     """Get the default path to the Visual Genome image directory.
 
     Returns:
         str: A path to the image directory.
 
     """
-
     def move_files(src_dir, dst_dir):
         # Move all files in the src_dir to the dst_dir and remove the src_dir
         for f in os.listdir(src_dir):
@@ -50,7 +48,6 @@ def move_files(src_dir, dst_dir):
 
 
 def get_image_data():
-
     """Get the default path to the image data JSON file.
 
     Returns:
@@ -62,7 +59,6 @@ def get_image_data():
 
 
 def get_region_descriptions():
-
     """Get the default path to the region descriptions JSON file.
 
     Returns:
@@ -75,7 +71,6 @@ def get_region_descriptions():
 
 
 class VisualGenomeDatasetBase(chainer.dataset.DatasetMixin):
-
     """Base class for Visual Genome dataset.
 
     """

From 0f68690f22c03b3ed2e5fc624f48fdcb6c9dbf74 Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Thu, 1 Jun 2017 07:23:36 +0000
Subject: [PATCH 08/11] Fix PEP8

---
 chainercv/datasets/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/chainercv/datasets/__init__.py b/chainercv/datasets/__init__.py
index 6ff38e22c1..790b49d5a0 100644
--- a/chainercv/datasets/__init__.py
+++ b/chainercv/datasets/__init__.py
@@ -6,8 +6,8 @@
 from chainercv.datasets.cub.cub_label_dataset import CUBLabelDataset  # NOQA
 from chainercv.datasets.online_products.online_products_dataset import OnlineProductsDataset  # NOQA
 from chainercv.datasets.transform_dataset import TransformDataset  # NOQA
-from chainercv.datasets.visual_genome.visual_genome_utils import VisualGenomeDatasetBase  # NQDA
-from chainercv.datasets.visual_genome.visual_genome_region_descriptions_dataset import VisualGenomeRegionDescriptionsDataset  # NQDA
+from chainercv.datasets.visual_genome.visual_genome_region_descriptions_dataset import VisualGenomeRegionDescriptionsDataset  # NOQA
+from chainercv.datasets.visual_genome.visual_genome_utils import VisualGenomeDatasetBase  # NOQA
 from chainercv.datasets.voc.voc_detection_dataset import VOCDetectionDataset  # NOQA
 from chainercv.datasets.voc.voc_semantic_segmentation_dataset import VOCSemanticSegmentationDataset  # NOQA
 from chainercv.datasets.voc.voc_utils import voc_detection_label_names  # NOQA

From d0e18cd986c762c43a99434dfba4f95009f92cd2 Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Sat, 10 Jun 2017 18:02:05 +0000
Subject: [PATCH 09/11] Remove transform-related code

---
 .../visual_genome_region_descriptions_dataset.py          | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
index 48a40e9d0a..2f549bbf7e 100644
--- a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -14,8 +14,6 @@
 from chainercv.datasets.visual_genome.visual_genome_utils import root
 from chainercv.datasets.visual_genome.visual_genome_utils import \
     VisualGenomeDatasetBase
-from chainercv.transforms import resize
-from chainercv.transforms import resize_bbox
 
 
 def get_vocabulary(region_descriptions='auto', min_token_instances=15):
@@ -74,12 +72,6 @@ def get_example(self, i):
         regions = np.vstack(regions).astype(np.float32)
         phrases = np.vstack(phrases).astype(np.int32)
 
-        if self.img_size is not None:
-            h_orig, w_orig = img.shape[1:]
-            img = resize(img, self.img_size)
-            h, w = img.shape[1:]
-            regions = resize_bbox(regions, (w_orig, h_orig), (w, h))
-
         return img, regions, phrases
 
 

From 22d315373ca47baeb4fa2a7494562db8f3370f8e Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Wed, 26 Jul 2017 12:16:38 +0000
Subject: [PATCH 10/11] Prepend with sos and pad with eos

---
 ...sual_genome_region_descriptions_dataset.py | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
index 2f549bbf7e..a32a1205b9 100644
--- a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -102,7 +102,7 @@ def loader(path):
 
 
 def _get_regions(region_descriptions_path):
-    """Region ID (int) -> Region bounding box (tuple of int).
+    """Region ID (int) -> Region bounding box (xmin, ymin).
 
     """
     data_root = download.get_dataset_directory(root)
@@ -116,11 +116,11 @@ def creator(path):
             for region_description in region_descriptions:
                 for region in region_description['regions']:
                     region_id = region['region_id']  # int
-                    xmin = region['x']
-                    ymin = region['y']
-                    xmax = xmin + region['width']
-                    ymax = ymin + region['height']
-                    regions[region_id] = (xmin, ymin, xmax, ymax)
+                    x_min = region['x']
+                    y_min = region['y']
+                    x_max = x_min + region['width']
+                    y_max = y_min + region['height']
+                    regions[region_id] = (y_min, x_min, y_max, x_max)
         pickle.dump(regions, open(base_path, 'wb'))
         return regions
 
@@ -151,9 +151,12 @@ def creator(path):
                     region_id = region['region_id']
                     tokens = _preprocess_phrase(region['phrase']).split()
                     if max_token_length > 0 and \
-                            len(tokens) <= max_token_length:
-                        phrase = np.zeros(max_token_length, dtype=np.int32)
-                        for i, token in enumerate(tokens):
+                            len(tokens) < max_token_length - 1:
+                        # <sos>, t1, t2,..., tn, <eos>, <eos>,..., <eos>
+                        phrase = np.empty(max_token_length, dtype=np.int32)
+                        phrase.fill(vocab['<eos>'])
+                        phrase[0] = vocab['<sos>']
+                        for i, token in enumerate(tokens, 1):
                             if token not in vocab:
                                 token = '<unk>'
                             token_id = vocab[token]
@@ -217,12 +220,14 @@ def _load_words(region_descriptions_path, min_token_instances):
     for word, count in six.iteritems(word_counts):
         if min_token_instances is None or count >= min_token_instances:
             words.append(word)
-    words.append('<unk>')
-    words.append('<eos>')
 
     # Sort to make sure that word orders are consistent
     words = sorted(words)
 
+    words.insert(0, '<unk>')
+    words.insert(0, '<eos>')
+    words.insert(0, '<sos>')
+
     return words
 
 

From 8676f5a3ea240e858fb8b8c14ed0cd9ee3cf1cd6 Mon Sep 17 00:00:00 2001
From: hvy <hiroyuki.vincent.yamazaki@gmail.com>
Date: Wed, 26 Jul 2017 16:02:57 +0000
Subject: [PATCH 11/11] Change sos to bos

---
 .../visual_genome_region_descriptions_dataset.py            | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
index a32a1205b9..2167bf6a11 100644
--- a/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
+++ b/chainercv/datasets/visual_genome/visual_genome_region_descriptions_dataset.py
@@ -152,10 +152,10 @@ def creator(path):
                     tokens = _preprocess_phrase(region['phrase']).split()
                     if max_token_length > 0 and \
                             len(tokens) < max_token_length - 1:
-                        # <sos>, t1, t2,..., tn, <eos>, <eos>,..., <eos>
+                        # <bos>, t1, t2,..., tn, <eos>, <eos>,..., <eos>
                         phrase = np.empty(max_token_length, dtype=np.int32)
                         phrase.fill(vocab['<eos>'])
-                        phrase[0] = vocab['<sos>']
+                        phrase[0] = vocab['<bos>']
                         for i, token in enumerate(tokens, 1):
                             if token not in vocab:
                                 token = '<unk>'
@@ -226,7 +226,7 @@ def _load_words(region_descriptions_path, min_token_instances):
 
     words.insert(0, '<unk>')
     words.insert(0, '<eos>')
-    words.insert(0, '<sos>')
+    words.insert(0, '<bos>')
 
     return words