Merge pull request #304 from Parry-Parry/master

feat: add DL 2023 qrels for MSMARCO passage and document v2

Author: seanmacavaney

ir_datasets/datasets/msmarco_document_v2.py

@@ -222,8 +222,15 @@ def _init():
     subsets['trec-dl-2023'] = Dataset(
         collection,
         TsvQueries(dlc['trec-dl-2023/queries'], namespace='msmarco', lang='en'),
+        TrecQrels(dlc['trec-dl-2023/qrels'], TREC_DL_QRELS_DEFS),
         TrecScoredDocs(GzipExtract(dlc['trec-dl-2023/scoreddocs'])),
     )
+    dl23_judged = Lazy(lambda: {q.query_id for q in subsets['trec-dl-2023'].qrels_iter()})
+    subsets['trec-dl-2023/judged'] = Dataset(
+        FilteredQueries(subsets['trec-dl-2023'].queries_handler(), dl23_judged),
+        FilteredScoredDocs(subsets['trec-dl-2023'].scoreddocs_handler(), dl23_judged),
+        subsets['trec-dl-2023'],
+    )
 
     subsets['anchor-text'] = Dataset(
         MsMarcoV2AnchorTextDocs(

ir_datasets/datasets/msmarco_passage_v2.py

@@ -316,8 +316,15 @@ def _init():
     subsets['trec-dl-2023'] = Dataset(
         collection,
         TsvQueries(dlc['trec-dl-2023/queries'], namespace='msmarco', lang='en'),
+        TrecQrels(dlc['trec-dl-2023/qrels'], TREC_DL_QRELS_DEFS),
         TrecScoredDocs(GzipExtract(dlc['trec-dl-2023/scoreddocs'])),
     )
+    dl23_judged = Lazy(lambda: {q.query_id for q in subsets['trec-dl-2023'].qrels_iter()})
+    subsets['trec-dl-2023/judged'] = Dataset(
+        FilteredQueries(subsets['trec-dl-2023'].queries_handler(), dl23_judged),
+        FilteredScoredDocs(subsets['trec-dl-2023'].scoreddocs_handler(), dl23_judged),
+        subsets['trec-dl-2023'],
+    )
 
     ir_datasets.registry.register(NAME, Dataset(collection, documentation("_")))
     for s in sorted(subsets):

ir_datasets/docs/msmarco-document-v2.yaml

@@ -130,6 +130,18 @@ trec-dl-2023:
 <p>
 Official topics for the TREC Deep Learning (DL) 2023 shared task.
 </p>
+<p>
+Note that these qrels are <i>inferred</i> from the passage ranking task; a document''s relevance
+label is the maximum of the labels of its passages.
+</p>
+'
+
+trec-dl-2023/judged:
+  desc: '
+<p>
+<a class="ds-ref">msmarco-document-v2/trec-dl-2023</a>, but filtered down to only the queries
+with qrels.
+</p>
 '
 
 anchor-text:

ir_datasets/docs/msmarco-passage-v2.yaml

@@ -104,4 +104,18 @@ trec-dl-2023:
 <p>
 Official topics for the TREC Deep Learning (DL) 2023 shared task.
 </p>
+<p>
+Note that the officially-released qrels <i>include</i> relevance labels propagated to
+duplicate passages, while results presented in the notebook papers remove duplicate documents.
+This means that the results are not directly comparable, and extra care should be taken when
+making comparisons among systems to ensure that they were evaluated in the same settings.
+</p>
+'
+
+trec-dl-2023/judged:
+  desc: '
+<p>
+<a class="ds-ref">msmarco-passage-v2/trec-dl-2023</a>, but filtered down to only the queries
+with qrels.
+</p>
 '

ir_datasets/etc/downloads.json

@@ -4711,9 +4711,16 @@
       "size_hint": 902168,
       "expected_md5": "0f5d548e53afb9e319c837ad67f9046a",
       "cache_path": "trec-dl-2023/top100.txt.gz"
+    },
+    "trec-dl-2023/qrels": {
+      "url": "https://trec.nist.gov/data/deep/2023.qrels.docs.withDupes.txt",
+      "irds_mirror": true,
+      "size_hint": 675015,
+      "expected_md5": "1e9c540b3cb03bcc975a583586c04090",
+      "cache_path": "trec-dl-2023/qrels.withDupes.txt"
     }
   },
-  
+
   "msmarco-passage": {
     "collectionandqueries": {
       "url": "https://msmarco.z22.web.core.windows.net/msmarcoranking/collectionandqueries.tar.gz",
@@ -4939,6 +4946,13 @@
       "expected_md5": "c339ed75e1556cacb387899f34cadad1",
       "cache_path": "trec-dl-2023/top100.txt.gz"
     },
+    "trec-dl-2023/qrels": {
+      "url": "https://trec.nist.gov/data/deep/2023.qrels.pass.withDupes.txt",
+      "irds_mirror": true,
+      "size_hint": 912450,
+      "expected_md5": "3a742d51ae65da2ece9c09b304b9e358",
+      "cache_path": "trec-dl-2023/qrels.withDupes.txt"
+    },
     "dedup_positions": {
       "url": "https://huggingface.co/datasets/macavaney/msmarco-passage-v2-duplicate-ids/resolve/main/dedup_positions.tar.gz",
       "size_hint": 388936118,

ir_datasets/etc/metadata.json

@@ -509,7 +509,8 @@
   "msmarco-document-v2/trec-dl-2021/judged": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 57}, "qrels": {"_ref": "msmarco-document-v2/trec-dl-2021"}, "scoreddocs": {"count": 5700}},
   "msmarco-document-v2/trec-dl-2022": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 500}, "scoreddocs": {"count": 50000}, "qrels": {"count": 369638, "fields": {"relevance": {"counts_by_value": {"0": 274971, "1": 49016, "2": 44255, "3": 1396}}}}},
   "msmarco-document-v2/trec-dl-2022/judged": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 76}, "qrels": {"_ref": "msmarco-document-v2/trec-dl-2022"}, "scoreddocs": {"count": 7600}},
-  "msmarco-document-v2/trec-dl-2023": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 700}, "scoreddocs": {"count": 70000}},
+  "msmarco-document-v2/trec-dl-2023": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 700}, "scoreddocs": {"count": 70000}, "qrels": {"count": 18034, "fields": {"relevance": {"counts_by_value": {"0": 11876, "1": 3015, "2": 1668, "3": 1475}}}}},
+  "msmarco-document-v2/trec-dl-2023/judged": {"docs": {"_ref": "msmarco-document-v2"}, "queries": {"count": 82}, "qrels": {"_ref": "msmarco-document-v2/trec-dl-2023"}, "scoreddocs": {"count": 8200}},
   "msmarco-document/anchor-text": {"docs": {"count": 1703834, "fields": {"doc_id": {"max_len": 8, "common_prefix": "D"}}}},
   "msmarco-document/dev": {"docs": {"_ref": "msmarco-document"}, "queries": {"count": 5193}, "qrels": {"count": 5193, "fields": {"relevance": {"counts_by_value": {"1": 5193}}}}, "scoreddocs": {"count": 519300}},
   "msmarco-document/eval": {"docs": {"_ref": "msmarco-document"}, "queries": {"count": 5793}, "scoreddocs": {"count": 579300}},
@@ -535,7 +536,8 @@
   "msmarco-passage-v2/trec-dl-2021/judged": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 53}, "qrels": {"_ref": "msmarco-passage-v2/trec-dl-2021"}, "scoreddocs": {"count": 5300}},
   "msmarco-passage-v2/trec-dl-2022": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 500}, "scoreddocs": {"count": 50000}, "qrels": {"count": 386416, "fields": {"relevance": {"counts_by_value": {"0": 286459, "1": 52218, "2": 46080, "3": 1659}}}}},
   "msmarco-passage-v2/trec-dl-2022/judged": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 76}, "qrels": {"_ref": "msmarco-passage-v2/trec-dl-2022"}, "scoreddocs": {"count": 7600}},
-  "msmarco-passage-v2/trec-dl-2023": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 700}, "scoreddocs": {"count": 70000}},
+  "msmarco-passage-v2/trec-dl-2023": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 700}, "scoreddocs": {"count": 70000}, "qrels": {"count": 22327, "fields": {"relevance": {"counts_by_value": {"0": 13866, "1": 4372, "2": 2259, "3": 1830}}}}},
+  "msmarco-passage-v2/trec-dl-2023/judged": {"docs": {"_ref": "msmarco-passage-v2"}, "queries": {"count": 82}, "qrels": {"_ref": "msmarco-passage-v2/trec-dl-2023"}, "scoreddocs": {"count": 8200}},
   "msmarco-passage/dev": {"docs": {"_ref": "msmarco-passage"}, "queries": {"count": 101093}, "qrels": {"count": 59273, "fields": {"relevance": {"counts_by_value": {"1": 59273}}}}},
   "msmarco-passage/dev/2": {"docs": {"_ref": "msmarco-passage"}, "queries": {"count": 4281}, "qrels": {"count": 4655, "fields": {"relevance": {"counts_by_value": {"1": 4655}}}}},
   "msmarco-passage/dev/judged": {"docs": {"_ref": "msmarco-passage"}, "queries": {"count": 55578}, "qrels": {"_ref": "msmarco-passage/dev"}},

test/integration/msmarco_document_v2.py

@@ -74,6 +74,11 @@ def test_queries(self):
             9: GenericQuery('2001686', 'good food and bad food for high cholesterol'),
             699: GenericQuery('3100949', 'How do birth control and hormone levels affect menstrual cycle variations?'),
         })
+        self._test_queries('msmarco-document-v2/trec-dl-2023/judged', count=82, items={
+            0: GenericQuery('2001010', 'cost comparison of funerals in australia'),
+            9: GenericQuery('2003787', 'how to make linkedin private'),
+            81: GenericQuery('3100922', 'What is the meaning and origin of the name Corrin?'),
+        })
 
     def test_qrels(self):
         self._test_qrels('msmarco-document-v2/train', count=331956, items={
@@ -131,6 +136,16 @@ def test_qrels(self):
             9: TrecQrel('2000511', 'msmarco_doc_00_928870217', 0, '0'),
             369637: TrecQrel('2056323', 'msmarco_doc_59_419476385', 1, '0'),
         })
+        self._test_qrels('msmarco-document-v2/trec-dl-2023', count=18034, items={
+            0: TrecQrel('2001010', 'msmarco_doc_00_1413647782', 0, '0'),
+            9: TrecQrel('2001010', 'msmarco_doc_01_710280219', 0, '0'),
+            18033: TrecQrel('3100922', 'msmarco_doc_58_1542021309', 0, '0'),
+        })
+        self._test_qrels('msmarco-document-v2/trec-dl-2023/judged', count=18034, items={
+            0: TrecQrel('2001010', 'msmarco_doc_00_1413647782', 0, '0'),
+            9: TrecQrel('2001010', 'msmarco_doc_01_710280219', 0, '0'),
+            18033: TrecQrel('3100922', 'msmarco_doc_58_1542021309', 0, '0'),
+        })
 
     def test_scoreddocs(self):
         self._test_scoreddocs('msmarco-document-v2/train', count=32218809, items={
@@ -168,6 +183,11 @@ def test_scoreddocs(self):
             9: GenericScoredDoc('2000138', 'msmarco_doc_50_2209372409', 13.6826),
             69999: GenericScoredDoc('3100949', 'msmarco_doc_42_771008717', 18.752998),
         })
+        self._test_scoreddocs('msmarco-document-v2/trec-dl-2023/judged', count=8200, items={
+            0: GenericScoredDoc('2001010', 'msmarco_doc_40_566250732', 11.2369),
+            9: GenericScoredDoc('2001010', 'msmarco_doc_41_285574609', 9.2133),
+            8199: GenericScoredDoc('3100922', 'msmarco_doc_19_2622779171', 9.289),
+        })
 
     def test_anchor_text(self):
         self._test_docs("msmarco-document-v2/anchor-text", count=4821244, items={

test/integration/msmarco_passage_v2.py

@@ -65,6 +65,11 @@ def test_queries(self):
             9: GenericQuery('2001686', 'good food and bad food for high cholesterol'),
             699: GenericQuery('3100949', 'How do birth control and hormone levels affect menstrual cycle variations?'),
         })
+        self._test_queries('msmarco-passage-v2/trec-dl-2023/judged', count=82, items={
+            0: GenericQuery('2001010', 'cost comparison of funerals in australia'),
+            9: GenericQuery('2003787', 'how to make linkedin private'),
+            81: GenericQuery('3100922', 'What is the meaning and origin of the name Corrin?'),
+        })
 
     def test_qrels(self):
         self._test_qrels('msmarco-passage-v2/train', count=284212, items={
@@ -102,6 +107,16 @@ def test_qrels(self):
             9: TrecQrel('2000511', 'msmarco_passage_00_491585086', 0, '0'),
             386415: TrecQrel('2056323', 'msmarco_passage_68_715747739', 1, '0'),
         })
+        self._test_qrels('msmarco-passage-v2/trec-dl-2023', count=22327, items={
+            0: TrecQrel('2001010', 'msmarco_passage_00_257661787', 0, '0'),
+            9: TrecQrel('2001010', 'msmarco_passage_01_221183941', 0, '0'),
+            22326: TrecQrel('3100922', 'msmarco_passage_68_194985280', 0, '0'),
+        })
+        self._test_qrels('msmarco-passage-v2/trec-dl-2023/judged', count=22327, items={
+            0: TrecQrel('2001010', 'msmarco_passage_00_257661787', 0, '0'),
+            9: TrecQrel('2001010', 'msmarco_passage_01_221183941', 0, '0'),
+            22326: TrecQrel('3100922', 'msmarco_passage_68_194985280', 0, '0'),
+        })
 
     def test_scoreddocs(self):
         self._test_scoreddocs('msmarco-passage-v2/train', count=27713673, items={
@@ -139,6 +154,11 @@ def test_scoreddocs(self):
             9: GenericScoredDoc('2000138', 'msmarco_passage_35_358067216', 15.2805),
             69999: GenericScoredDoc('3100949', 'msmarco_passage_30_84437641', 18.801701),
         })
+        self._test_scoreddocs('msmarco-passage-v2/trec-dl-2023/judged', count=8200, items={
+            0: GenericScoredDoc('2001010', 'msmarco_passage_39_224640845', 11.9941),
+            9: GenericScoredDoc('2001010', 'msmarco_passage_60_353412311', 10.992799),
+            8199: GenericScoredDoc('3100922', 'msmarco_passage_38_636536351', 10.032599),
+        })
 
 
 if __name__ == '__main__':