From 8da5a850f300a9a7da90b984352360766d7fe6fa Mon Sep 17 00:00:00 2001 From: Natasha Seelam Date: Sat, 4 Jun 2022 00:50:22 -0400 Subject: [PATCH 1/3] fix biosses for pearson r --- .../biosses/biosses_bigbio_pairs/templates.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml b/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml index 695678191..61ddc928b 100644 --- a/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml +++ b/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml @@ -17,7 +17,7 @@ templates: name: bigbio_sts_seem_binary reference: '' 9c87c669-7be0-42f3-bd38-32ff149f2722: !Template - answer_choices: null + answer_choices: 0 ||| 1 ||| 2 ||| 3 ||| 4 id: 9c87c669-7be0-42f3-bd38-32ff149f2722 jinja: 'from {{"0.0"}} to {{"4.0"}}, how similar are "{{text_1}}" and "{{text_2}}"?||| @@ -30,7 +30,7 @@ templates: name: bigbio_sts_similarity_scale reference: '' abc04725-8f0d-4653-a813-b9b41a45f253: !Template - answer_choices: null + answer_choices: 0 ||| 1 ||| 2 ||| 3 ||| 4 id: abc04725-8f0d-4653-a813-b9b41a45f253 jinja: 'How similar are "{{text_1}}" and "{{text_2}}"? Give a score between {{"0.0"}} and {{"4.0"}}.||| @@ -38,7 +38,8 @@ templates: {{label}}' metadata: !TemplateMetadata choices_in_prompt: false - metrics: [] + metrics: + - Accuracy original_task: false name: bigbio_sts_similarity_how reference: '' @@ -56,7 +57,7 @@ templates: name: bigbio_sts_express_binary reference: '' e0b0f20b-47a8-459d-898e-d9a428706e20: !Template - answer_choices: null + answer_choices: 0 ||| 1 ||| 2 ||| 3 ||| 4 id: e0b0f20b-47a8-459d-898e-d9a428706e20 jinja: 'Rate the similarity of these two sentences ({{"0.0"}} being the lowest and {{"4.0"}} the highest): @@ -68,6 +69,7 @@ templates: choices_in_prompt: false metrics: - Pearson Correlation + - Accuracy original_task: false name: bigbio_sts_similarity_rate reference: '' From 3ea86b2bd0e29348dbc5205e96c2dbb0fdf584f6 Mon Sep 17 00:00:00 2001 From: Natasha Seelam Date: Sat, 4 Jun 2022 22:26:21 -0400 Subject: [PATCH 2/3] remove all non-accuracy metrics and fix answer choices --- .../templates/biosses/biosses_bigbio_pairs/templates.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml b/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml index 61ddc928b..3cce7f0e9 100644 --- a/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml +++ b/promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml @@ -25,7 +25,7 @@ templates: metadata: !TemplateMetadata choices_in_prompt: false metrics: - - Pearson Correlation + - Accuracy original_task: false name: bigbio_sts_similarity_scale reference: '' @@ -68,7 +68,6 @@ templates: metadata: !TemplateMetadata choices_in_prompt: false metrics: - - Pearson Correlation - Accuracy original_task: false name: bigbio_sts_similarity_rate From d04b4e9034c0ec83cec68e1e5bc3d4222b05b1ff Mon Sep 17 00:00:00 2001 From: Natasha Seelam Date: Sat, 4 Jun 2022 23:15:10 -0400 Subject: [PATCH 3/3] adds blurb split from fold1 for biosses --- .../gad/gad_blurb_bigbio_text/templates.yaml | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 promptsource/templates/gad/gad_blurb_bigbio_text/templates.yaml diff --git a/promptsource/templates/gad/gad_blurb_bigbio_text/templates.yaml b/promptsource/templates/gad/gad_blurb_bigbio_text/templates.yaml new file mode 100644 index 000000000..c122eb1b0 --- /dev/null +++ b/promptsource/templates/gad/gad_blurb_bigbio_text/templates.yaml @@ -0,0 +1,103 @@ +dataset: gad +subset: gad_fold1_bigbio_text +templates: + 079471a6-8eb1-468c-ad98-b40d9e290caf: !Template + answer_choices: No ||| Yes + id: 079471a6-8eb1-468c-ad98-b40d9e290caf + jinja: 'Is there an association between the gene @GENE$ and the disease @DISEASE$ + expressed in this passage? + + + {{ text }} + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: Is there an association expressed? (passage last) + reference: '"Other" metric refers to the micro F1 used in BLURB.' + 8d5b31ab-3eb3-4f6d-a653-3ec1c4704f39: !Template + answer_choices: No ||| Yes + id: 8d5b31ab-3eb3-4f6d-a653-3ec1c4704f39 + jinja: 'I''m a doctor. Can you tell me, is there an association between the gene + @GENE$ and the disease @DISEASE$ expressed in this passage? + + + {{ text }} + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: I'm a doctor + reference: '"Other" metric refers to the micro F1 used in BLURB.' + 9ecf58f5-e6dd-486e-8234-f050fa806d85: !Template + answer_choices: No ||| Yes + id: 9ecf58f5-e6dd-486e-8234-f050fa806d85 + jinja: '{{ text }} + + + Does this passage indicate that there is an association between the gene @GENE$ + and the disease @DISEASE$ ? + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: Does this passage (passage first) + reference: '"Other" metric refers to the micro F1 used in BLURB.' + b26032d2-3edd-4dd9-b1d3-379db0d7f145: !Template + answer_choices: No ||| Yes + id: b26032d2-3edd-4dd9-b1d3-379db0d7f145 + jinja: 'Does the following passage indicate that there is an association between + the gene @GENE$ and the disease @DISEASE$ ? + + + {{ text }} + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: Does this passage (passage last) + reference: '"Other" metric refers to the micro F1 used in BLURB.' + edf792c0-6598-4e41-aa38-8841561b203f: !Template + answer_choices: No ||| Yes + id: edf792c0-6598-4e41-aa38-8841561b203f + jinja: '{{ text }} + + + Is there an association between the gene @GENE$ and the disease @DISEASE$ expressed + in this passage? + + ||| + + {{ answer_choices[labels[0] | int] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + - Other + original_task: true + name: Is there an association expressed? (passage first) + reference: '"Other" metric refers to the micro F1 used in BLURB.'