Source code for torchnlp.datasets.iwslt

import os
import xml.etree.ElementTree as ElementTree
import io
import glob

from import download_file_maybe_extract

[docs]def iwslt_dataset( directory='data/iwslt/', train=False, dev=False, test=False, language_extensions=['en', 'de'], train_filename='{source}-{target}/train.{source}-{target}.{lang}', dev_filename='{source}-{target}/IWSLT16.TED.tst2013.{source}-{target}.{lang}', test_filename='{source}-{target}/IWSLT16.TED.tst2014.{source}-{target}.{lang}', check_files=['{source}-{target}/train.tags.{source}-{target}.{source}'], url='{source}/{target}/{source}-{target}.tgz'): """ Load the International Workshop on Spoken Language Translation (IWSLT) 2017 translation dataset. In-domain training, development and evaluation sets were supplied through the website of the WIT3 project, while out-of-domain training data were linked in the workshop’s website. With respect to edition 2016 of the evaluation campaign, some of the talks added to the TED repository during the last year have been used to define the evaluation sets (tst2017), while the remaining new talks have been included in the training sets. The English data that participants were asked to recognize and translate consists in part of TED talks as in the years before, and in part of real-life lectures and talks that have been mainly recorded in lecture halls at KIT and Carnegie Mellon University. TED talks are challenging due to their variety in topics, but are very benign as they are very thoroughly rehearsed and planned, leading to easy to recognize and translate language. Note: The order examples are returned is not guaranteed due to ``iglob``. References: * * **Citation:** M. Cettolo, C. Girardi, and M. Federico. 2012. WIT3: Web Inventory of Transcribed and Translated Talks. In Proc. of EAMT, pp. 261-268, Trento, Italy. Args: directory (str, optional): Directory to cache the dataset. train (bool, optional): If to load the training split of the dataset. dev (bool, optional): If to load the dev split of the dataset. test (bool, optional): If to load the test split of the dataset. language_extensions (:class:`list` of :class:`str`): Two language extensions ['en'|'de'|'it'|'ni'|'ro'] to load. train_filename (str, optional): The filename of the training split. dev_filename (str, optional): The filename of the dev split. test_filename (str, optional): The filename of the test split. check_files (str, optional): Check if these files exist, then this download was successful. url (str, optional): URL of the dataset file. Returns: :class:`tuple` of :class:`iterable` or :class:`iterable`: Returns between one and all dataset splits (train, dev and test) depending on if their respective boolean argument is ``True``. Example: >>> from torchnlp.datasets import iwslt_dataset # doctest: +SKIP >>> train = iwslt_dataset(train=True) # doctest: +SKIP >>> train[:2] # doctest: +SKIP [{ 'en': "David Gallo: This is Bill Lange. I'm Dave Gallo.", 'de': 'David Gallo: Das ist Bill Lange. Ich bin Dave Gallo.' }, { 'en': "And we're going to tell you some stories from the sea here in video.", 'de': 'Wir werden Ihnen einige Geschichten über das Meer in Videoform erzählen.' }] """ if len(language_extensions) != 2: raise ValueError("`language_extensions` must be two language extensions " "['en'|'de'|'it'|'ni'|'ro'] to load.") # Format Filenames source, target = tuple(language_extensions) check_files = [s.format(source=source, target=target) for s in check_files] url = url.format(source=source, target=target) download_file_maybe_extract(url=url, directory=directory, check_files=check_files) iwslt_clean(os.path.join(directory, '{source}-{target}'.format(source=source, target=target))) ret = [] splits = [(train, train_filename), (dev, dev_filename), (test, test_filename)] splits = [f for (requested, f) in splits if requested] for filename in splits: examples = [] for extension in language_extensions: path = os.path.join(directory, filename.format(lang=extension, source=source, target=target)) with open(path, 'r', encoding='utf-8') as f: language_specific_examples = [l.strip() for l in f] if len(examples) == 0: examples = [{} for _ in range(len(language_specific_examples))] for i, example in enumerate(language_specific_examples): examples[i][extension] = example ret.append(examples) if len(ret) == 1: return ret[0] else: return tuple(ret)
def iwslt_clean(directory): # Thanks to torchtext for this snippet: # for xml_filename in glob.iglob(os.path.join(directory, '*.xml')): txt_filename = os.path.splitext(xml_filename)[0] if os.path.isfile(txt_filename): continue with, mode='w', encoding='utf-8') as f: root = ElementTree.parse(xml_filename).getroot()[0] for doc in root.findall('doc'): for element in doc.findall('seg'): f.write(element.text.strip() + '\n') xml_tags = [ '<url', '<keywords', '<talkid', '<description', '<reviewer', '<translator', '<title', '<speaker' ] for original_filename in glob.iglob(os.path.join(directory, 'train.tags*')): txt_filename = original_filename.replace('.tags', '') if os.path.isfile(txt_filename): continue with, mode='w', encoding='utf-8') as txt_file, \, mode='r', encoding='utf-8') as original_file: for line in original_file: if not any(tag in line for tag in xml_tags): txt_file.write(line.strip() + '\n')