FLUB_croissant_metadata.json

{
  "@context": {
    "@language": "en",
    "@vocab": "https://schema.org/",
    "citeAs": "cr:citeAs",
    "column": "cr:column",
    "conformsTo": "dct:conformsTo",
    "cr": "http://mlcommons.org/croissant/",
    "rai": "http://mlcommons.org/croissant/RAI/",
    "data": {
      "@id": "cr:data",
      "@type": "@json"
    },
    "dataType": {
      "@id": "cr:dataType",
      "@type": "@vocab"
    },
    "dct": "http://purl.org/dc/terms/",
    "examples": {
      "@id": "cr:examples",
      "@type": "@json"
    },
    "extract": "cr:extract",
    "field": "cr:field",
    "fileProperty": "cr:fileProperty",
    "fileObject": "cr:fileObject",
    "fileSet": "cr:fileSet",
    "format": "cr:format",
    "includes": "cr:includes",
    "isLiveDataset": "cr:isLiveDataset",
    "jsonPath": "cr:jsonPath",
    "key": "cr:key",
    "md5": "cr:md5",
    "parentField": "cr:parentField",
    "path": "cr:path",
    "recordSet": "cr:recordSet",
    "references": "cr:references",
    "regex": "cr:regex",
    "repeated": "cr:repeated",
    "replace": "cr:replace",
    "sc": "https://schema.org/",
    "separator": "cr:separator",
    "source": "cr:source",
    "subField": "cr:subField",
    "transform": "cr:transform"
  },
  "@type": "sc:Dataset",
  "name": "FLUB",
  "description": "In this paper, we challenge the reasoning and understanding abilities of LLMs by proposing a FaLlacy Understanding Benchmark (FLUB) containing cunning texts that are easy for humans to understand but difficult for models to grasp. Specifically, the cunning texts that FLUB focuses on mainly consist of the tricky, humorous, and misleading texts collected from the real internet environment. FLUB has 8 fine-grained types of cunning texts and most of the texts in FLUB fall into two types of fallacy, namely, faulty reasoning and word game. Moreover, we also manually annotated one correct answer (i.e., the explanation of the cunning text) and three confusing wrong answers for each input text in FLUB. Based on FLUB and its annotation information, we design three tasks with increasing difficulty to test whether the LLMs can understand the fallacy and solve the ``cunning'' texts. Specifically, (1) Answer Selection: The model is asked to select the correct one from the four answers provided by FLUB for each input text. (2) Cunning Type Classification: Given a cunning text as input, the model is expected to directly identify its fallacy type defined in our scheme. (3) Fallacy Explanation: We hope the model sees a cunning text and intelligently generates a correct explanation for the fallacy contained in the text, just like humans, without falling into its trap.",
  "conformsTo": "http://mlcommons.org/croissant/1.0",
  "citeAs": "@article{li2024llms,title={When llms meet cunning texts: A fallacy understanding benchmark for large language models}, author={Li, Yinghui and Zhou, Qingyu and Luo, Yuanzhen and Ma, Shirong and Li, Yangning and Zheng, Hai-Tao and Hu, Xuming and Yu, Philip S}, journal={arXiv preprint arXiv:2402.11100}, year={2024}}",
  "url": "https://github.com/THUKElab/FLUB",
  "distribution": [
    {
      "@type": "cr:FileObject",
      "@id": "github-repository",
      "name": "github-repository",
      "description": "Tsinghua Knowledge Engineering Laboratory (SZ)",
      "contentUrl": "https://github.com/THUKElab/FLUB",
      "encodingFormat": "git+https",
      "sha256": "main"
    },
    {
      "@type": "cr:FileSet",
      "@id": "jsonl-files",
      "name": "jsonl-files",
      "description": "JSONL files are hosted on the GitHub repository.",
      "containedIn": {
        "@id": "github-repository"
      },
      "encodingFormat": "application/jsonl",
      "includes": "data/*.jsonl"
    }
  ],
  "recordSet": [
    {
      "@type": "cr:RecordSet",
      "@id": "jsonl",
      "name": "jsonl",
      "field": [
        {
          "@type": "cr:Field",
          "@id": "jsonl/id",
          "name": "id",
          "description": "The id of each data sample.",
          "dataType": "sc:Text",
          "source": {
            "fileSet": {
              "@id": "jsonl-files"
            },
            "extract": {
              "column": "id"
            }
          }
        },
        {
          "@type": "cr:Field",
          "@id": "jsonl/text",
          "name": "text",
          "description": "The input cunning text.",
          "dataType": "sc:Text",
          "source": {
            "fileSet": {
              "@id": "jsonl-files"
            },
            "extract": {
              "column": "text"
            }
          }
        },
        {
          "@type": "cr:Field",
          "@id": "jsonl/is_question",
          "name": "is_question",
          "description": "Is the input cunning text a question?",
          "dataType": "sc:Boolean",
          "source": {
            "fileSet": {
              "@id": "jsonl-files"
            },
            "extract": {
              "column": "is_question"
            }
          }
        },
        {
          "@type": "cr:Field",
          "@id": "jsonl/type",
          "name": "type",
          "description": "The cunning type of the input text for the Cunning Type Classification task.",
          "dataType": "sc:Text",
          "source": {
            "fileSet": {
              "@id": "jsonl-files"
            },
            "extract": {
              "column": "type"
            }
          }
        },
        {
          "@type": "cr:Field",
          "@id": "jsonl/explanation",
          "name": "explanation",
          "description": "The correct explanation of the input text for the Fallacy Explanation task.",
          "dataType": "sc:Text",
          "source": {
            "fileSet": {
              "@id": "jsonl-files"
            },
            "extract": {
              "column": "explanation"
            }
          }
        },
        {
          "@type": "cr:Field",
          "@id": "jsonl/options",
          "name": "options",
          "description": "The candidate answers for the input text (question).",
          "dataType": "sc:Text",
          "source": {
            "fileSet": {
              "@id": "jsonl-files"
            },
            "extract": {
              "column": "options"
            }
          }
        },
        {
          "@type": "cr:Field",
          "@id": "jsonl/answer",
          "name": "answer",
          "description": "The correct answer for the Answer Selection (Multiple Choice) task.",
          "dataType": "sc:Text",
          "source": {
            "fileSet": {
              "@id": "jsonl-files"
            },
            "extract": {
              "column": "answer"
            }
          }
        }
      ]
    }
  ]
}