Skip to content

Commit 539b048

Browse files
authored
Fix dataframe CSV tests on Windows (#39563)
1 parent 3a02af8 commit 539b048

2 files changed

Lines changed: 9 additions & 8 deletions

File tree

sdks/python/apache_beam/dataframe/io.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -736,7 +736,7 @@ def open(self, file_handle):
736736
self.empty = self.header = self.footer = None
737737
if not self.binary:
738738
file_handle = TextIOWrapper(
739-
file_handle, encoding=self.kwargs.get("encoding", None))
739+
file_handle, encoding=self.kwargs.get("encoding", None), newline='')
740740
self.file_handle = file_handle
741741

742742
def write_to(self, df, file_handle=None):

sdks/python/apache_beam/dataframe/io_test.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import importlib
1919
import math
2020
import os
21-
import platform
2221
import shutil
2322
import tempfile
2423
import typing
@@ -65,9 +64,6 @@ class MyRow(typing.NamedTuple):
6564
value: int
6665

6766

68-
@unittest.skipIf(
69-
platform.system() == 'Windows',
70-
'https://github.com/apache/beam/issues/20642')
7167
class IOTest(unittest.TestCase):
7268
def setUp(self):
7369
self._temp_roots = []
@@ -431,6 +427,11 @@ def test_file_not_found(self):
431427

432428
def test_windowed_write(self):
433429
output = self.temp_dir()
430+
431+
def no_colon_file_naming(*args):
432+
file_name = fileio.default_file_naming('out.csv')(*args)
433+
return file_name.replace(':', '_')
434+
434435
with beam.Pipeline() as p:
435436
pc = (
436437
p | beam.Create([MyRow(timestamp=i, value=i % 3) for i in range(20)])
@@ -440,18 +441,18 @@ def test_windowed_write(self):
440441
beam.window.FixedWindows(10)).with_output_types(MyRow))
441442

442443
deferred_df = convert.to_dataframe(pc)
443-
deferred_df.to_csv(output + 'out.csv', index=False)
444+
deferred_df.to_csv(output, file_naming=no_colon_file_naming, index=False)
444445

445446
first_window_files = (
446447
f'{output}out.csv-'
447-
f'{datetime.utcfromtimestamp(0).isoformat()}*')
448+
f'{datetime.utcfromtimestamp(0).isoformat().replace(":", "_")}*')
448449
self.assertCountEqual(
449450
['timestamp,value'] + [f'{i},{i % 3}' for i in range(10)],
450451
set(self.read_all_lines(first_window_files, delete=True)))
451452

452453
second_window_files = (
453454
f'{output}out.csv-'
454-
f'{datetime.utcfromtimestamp(10).isoformat()}*')
455+
f'{datetime.utcfromtimestamp(10).isoformat().replace(":", "_")}*')
455456
self.assertCountEqual(
456457
['timestamp,value'] + [f'{i},{i%3}' for i in range(10, 20)],
457458
set(self.read_all_lines(second_window_files, delete=True)))

0 commit comments

Comments
 (0)