| from . import InputExample |
| import csv |
| import gzip |
| import os |
| import gzip |
|
|
| class PairedFilesReader(object): |
| """ |
| Reads in the a Pair Dataset, split in two files |
| """ |
| def __init__(self, filepaths): |
| self.filepaths = filepaths |
|
|
|
|
| def get_examples(self, max_examples=0): |
| """ |
| """ |
| fIns = [] |
| for filepath in self.filepaths: |
| fIn = gzip.open(filepath, 'rt', encoding='utf-8') if filepath.endswith('.gz') else open(filepath, encoding='utf-8') |
| fIns.append(fIn) |
|
|
| examples = [] |
|
|
| eof = False |
| while not eof: |
| texts = [] |
| for fIn in fIns: |
| text = fIn.readline() |
|
|
| if text == '': |
| eof = True |
| break |
|
|
| texts.append(text) |
|
|
| if eof: |
| break; |
|
|
| examples.append(InputExample(guid=str(len(examples)), texts=texts, label=1)) |
| if max_examples > 0 and len(examples) >= max_examples: |
| break |
|
|
| return examples |
