Biosses gad fixes

promptsource/templates/biosses/biosses_bigbio_pairs/templates.yaml

@@ -17,28 +17,29 @@ templates:
     name: bigbio_sts_seem_binary
     reference: ''
   9c87c669-7be0-42f3-bd38-32ff149f2722: !Template
-    answer_choices: null
+    answer_choices: 0 ||| 1 ||| 2 ||| 3 ||| 4
     id: 9c87c669-7be0-42f3-bd38-32ff149f2722
     jinja: 'from {{"0.0"}} to {{"4.0"}}, how similar are "{{text_1}}" and "{{text_2}}"?|||
 
       {{label}}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
-      - Pearson Correlation
+      - Accuracy
       original_task: false
     name: bigbio_sts_similarity_scale
     reference: ''
   abc04725-8f0d-4653-a813-b9b41a45f253: !Template
-    answer_choices: null
+    answer_choices: 0 ||| 1 ||| 2 ||| 3 ||| 4
     id: abc04725-8f0d-4653-a813-b9b41a45f253
     jinja: 'How similar are "{{text_1}}" and "{{text_2}}"? Give a score between {{"0.0"}}
       and {{"4.0"}}.|||
 
       {{label}}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
-      metrics: []
+      metrics:
+      - Accuracy
       original_task: false
     name: bigbio_sts_similarity_how
     reference: ''
@@ -56,7 +57,7 @@ templates:
     name: bigbio_sts_express_binary
     reference: ''
   e0b0f20b-47a8-459d-898e-d9a428706e20: !Template
-    answer_choices: null
+    answer_choices: 0 ||| 1 ||| 2 ||| 3 ||| 4
     id: e0b0f20b-47a8-459d-898e-d9a428706e20
     jinja: 'Rate the similarity of these two sentences ({{"0.0"}} being the lowest
       and {{"4.0"}} the highest):
@@ -67,7 +68,7 @@ templates:
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
-      - Pearson Correlation
+      - Accuracy
       original_task: false
     name: bigbio_sts_similarity_rate
     reference: ''

promptsource/templates/gad/gad_blurb_bigbio_text/templates.yaml

@@ -0,0 +1,103 @@
+dataset: gad
+subset: gad_fold1_bigbio_text
+templates:
+  079471a6-8eb1-468c-ad98-b40d9e290caf: !Template
+    answer_choices: No ||| Yes
+    id: 079471a6-8eb1-468c-ad98-b40d9e290caf
+    jinja: 'Is there an association between the gene @GENE$ and the disease @DISEASE$
+      expressed in this passage?
+
+
+      {{ text }}
+
+      |||
+
+      {{ answer_choices[labels[0] | int] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: Is there an association expressed? (passage last)
+    reference: '"Other" metric refers to the micro F1 used in BLURB.'
+  8d5b31ab-3eb3-4f6d-a653-3ec1c4704f39: !Template
+    answer_choices: No ||| Yes
+    id: 8d5b31ab-3eb3-4f6d-a653-3ec1c4704f39
+    jinja: 'I''m a doctor. Can you tell me, is there an association between the gene
+      @GENE$ and the disease @DISEASE$ expressed in this passage?
+
+
+      {{ text }}
+
+      |||
+
+      {{ answer_choices[labels[0] | int] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: I'm a doctor
+    reference: '"Other" metric refers to the micro F1 used in BLURB.'
+  9ecf58f5-e6dd-486e-8234-f050fa806d85: !Template
+    answer_choices: No ||| Yes
+    id: 9ecf58f5-e6dd-486e-8234-f050fa806d85
+    jinja: '{{ text }}
+
+
+      Does this passage indicate that there is an association between the gene @GENE$
+      and the disease @DISEASE$ ?
+
+      |||
+
+      {{ answer_choices[labels[0] | int] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: Does this passage (passage first)
+    reference: '"Other" metric refers to the micro F1 used in BLURB.'
+  b26032d2-3edd-4dd9-b1d3-379db0d7f145: !Template
+    answer_choices: No ||| Yes
+    id: b26032d2-3edd-4dd9-b1d3-379db0d7f145
+    jinja: 'Does the following passage indicate that there is an association between
+      the gene @GENE$ and the disease @DISEASE$ ?
+
+
+      {{ text }}
+
+      |||
+
+      {{ answer_choices[labels[0] | int] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: Does this passage (passage last)
+    reference: '"Other" metric refers to the micro F1 used in BLURB.'
+  edf792c0-6598-4e41-aa38-8841561b203f: !Template
+    answer_choices: No ||| Yes
+    id: edf792c0-6598-4e41-aa38-8841561b203f
+    jinja: '{{ text }}
+
+
+      Is there an association between the gene @GENE$ and the disease @DISEASE$ expressed
+      in this passage?
+
+      |||
+
+      {{ answer_choices[labels[0] | int] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      - Other
+      original_task: true
+    name: Is there an association expressed? (passage first)
+    reference: '"Other" metric refers to the micro F1 used in BLURB.'