fix streaming from arrow files (#7083)

fschlatt · web-flow · commit ca58154bba18 · 2024-08-30T17:17:02.000+02:00
fix streaming from local arrow files
diff --git a/src/datasets/utils/file_utils.py b/src/datasets/utils/file_utils.py
@@ -480,6 +480,7 @@ def readline(f: io.RawIOBase):
     "pickle",
     "rel",
     "xml",
+    "arrow",
 ]
 COMPRESSION_EXTENSION_TO_PROTOCOL = {
     # single file compression

Original file line number	Diff line number	Diff line change
`@@ -480,6 +480,7 @@ def readline(f: io.RawIOBase):`
`480`	`480`	`"pickle",`
`481`	`481`	`"rel",`
`482`	`482`	`"xml",`
	`483`	`+ "arrow",`
`483`	`484`	`]`
`484`	`485`	`COMPRESSION_EXTENSION_TO_PROTOCOL = {`
`485`	`486`	`# single file compression`