import contextlib import os import unittest import io from sample_validator.iterators import BaseTXT, CSV, NDJSON, CSV_PARAMS, TXT from sample_validator.exceptions import TableFileStructError FIXTURES_PATH = os.path.join(os.path.dirname(__file__), "fixtures", "iterators") class IteratorTestCaseMixin: iterator_class = None iterator_kwargs = None iterator_correct_report = None iterator_incorrect_report = None iterator_empty_report = None @contextlib.contextmanager def get_file_obj(self, report_name): filepath = os.path.join(FIXTURES_PATH, report_name) with open(filepath, 'r') as fout: yield fout def test_correct(self): if not self.iterator_correct_report: self.skipTest('Name of correct report is not provided') with self.get_file_obj(self.iterator_correct_report) as fout: reader = self.iterator_class(file_obj=fout, **self.iterator_kwargs) for line in reader: self.assertTrue(type(line), dict) def test_incorrect(self): if not self.iterator_incorrect_report: self.skipTest('Name of incorrect report is not provided') with self.get_file_obj(self.iterator_incorrect_report) as fout: reader = self.iterator_class(file_obj=fout, **self.iterator_kwargs) self.assertRaises(TableFileStructError, next, reader) def test_empty(self): if not self.iterator_empty_report: self.skipTest('Name of empty report is not provided') with self.get_file_obj(self.iterator_empty_report) as fout: reader = self.iterator_class(file_obj=fout, **self.iterator_kwargs) self.assertListEqual(list(reader), []) class CSVIteratorTestCase(IteratorTestCaseMixin, unittest.TestCase): iterator_class = CSV iterator_kwargs = dict(rows_number=3, reader_params=CSV_PARAMS["tsv"]) iterator_correct_report = 'tsv_correct.tsv' iterator_incorrect_report = 'tsv_incorrect.tsv' iterator_empty_report = 'tsv_empty.tsv' class NDJSONIteratorTestCase(IteratorTestCaseMixin, unittest.TestCase): iterator_class = NDJSON iterator_kwargs = dict(rows_number=3) iterator_correct_report = 'ndjson_correct.ndjson' iterator_empty_report = 'ndjson_empty.ndjson' class TXTIteratorAmazonTestCase(IteratorTestCaseMixin, unittest.TestCase): iterator_class = TXT iterator_kwargs = dict(rows_number=5, delimiter='#*#', has_metadata=True) iterator_correct_report = 'txt_amazon_correct.txt' iterator_incorrect_report = 'txt_amazon_incorrect.txt' iterator_empty_report = 'txt_amazon_empty.txt' def test_correct_rows_counting_amazon(self): with self.get_file_obj('txt_amazon_correct.txt') as fout: reader = TXT(rows_number=5, file_obj=fout, delimiter='#*#', has_metadata=True) self.assertEqual(len(list(reader)), 5) def test_only_headers_amazon(self): with self.get_file_obj('txt_amazon_only_headers.txt') as fout: reader = TXT(rows_number=3, file_obj=fout, delimiter='#*#', has_metadata=True) self.assertFalse(list(reader)) class TXTIteratorSmeMaxTestCase(IteratorTestCaseMixin, unittest.TestCase): iterator_class = TXT iterator_kwargs = dict(rows_number=5, delimiter='\x07', header=["h1", "h2", "h3", "h4"]) iterator_correct_report = 'txt_sme_max_correct.txt' iterator_incorrect_report = 'txt_sme_max_incorrect.txt' iterator_empty_report = 'txt_sme_max_empty.txt' def test_correct_rows_counting(self): with self.get_file_obj('txt_sme_max_correct.txt') as fout: reader = TXT(rows_number=5, file_obj=fout, delimiter='\x07', header=["h1", "h2", "h3", "h4"]) self.assertEqual(len(list(reader)), 5) class CSVGRASIteratorTestCase(IteratorTestCaseMixin, unittest.TestCase): iterator_class = CSV iterator_kwargs = dict(rows_number=1000, reader_params={ 'escapechar': '\\', 'doublequote': False }) iterator_correct_report = 'csv_gras_correct.csv' class BaseTXTIteratorTestCase(unittest.TestCase): def test_readline_skip_empty(self): fileobj = io.StringIO('line 1\n\nline 2') reader = BaseTXT(file_obj=fileobj, rows_number=None, skip_empty=True) self.assertEqual(reader.readline(), 'line 1') self.assertEqual(reader.readline(), 'line 2') def test_backline(self): fileobj = io.StringIO('line 1\nline 2\nline 3') reader = BaseTXT(file_obj=fileobj, rows_number=None) reader.readline() reader.readline() reader.backline() self.assertEqual(reader.readline(), 'line 2') def test_is_eof(self): fileobj = io.StringIO('line 1\nline2') reader = BaseTXT(file_obj=fileobj, rows_number=None) self.assertFalse(reader.is_eof()) self.assertEqual(fileobj.tell(), 0) reader.readline() reader.readline() self.assertTrue(reader.is_eof()) self.assertTrue(fileobj.tell(), 11)