Add wikipedia_dataset

punitarani · punitarani · commit d1d02f0f6750 · 2023-05-13T16:59:40.000-07:00
diff --git a/rabbithole/wikipedia.py b/rabbithole/wikipedia.py
@@ -0,0 +1,8 @@
+"""rabbithole.wikipedia module"""
+
+from datasets import load_dataset
+
+wikipedia_dataset = load_dataset("Cohere/wikipedia-22-12-simple-embeddings", split="train", streaming=False)
+
+if __name__ == "__main__":
+    print(wikipedia_dataset.info)
