diff --git a/webpage/content/submission.md b/webpage/content/submission.md index 5819a575..dd658fb5 100644 --- a/webpage/content/submission.md +++ b/webpage/content/submission.md @@ -51,33 +51,6 @@ Submissions should be in `.json` format. ## 1.B Example Submissions \ -{{< collapse title="Information Retrieval sample submission file without optional fields." >}} -{ - "submission_name": "msmarco_bm_25", - "dataset_name": "MSMARCO", - "dataset_version": "0.0.1", - "embedding_name": "BM25", - "hparams": {}, - "packages": [], - "leaderboard_task_name": "Information Retrieval", - "metrics": [ - { - "NDCG@10": 31.50, - "MRR@10": 56.36 - } - ], - "metrics_avg": { - "NDCG@10": 31.50, - "MRR@10": 56.36 - }, - "metrics_std": { - "NDCG@10": 0.0, - "MRR@10": 0.0 - }, - "averaged_over": 1 -} -{{< /collapse >}} -\ {{< collapse title="Question Answering sample submission file with packages provided." >}} { @@ -221,13 +194,13 @@ submission.save_json() Example how to add dataset that is not currently in Leaderboard. To do that three files will be needed: - webpage/content/tasks/task{taksk_abrevation}.md -e.g.: [webpage/content/tasks/taskIR.md](https://raw.githubusercontent.com/CLARIN-PL/embeddings/main/webpage/content/tasks/taskIR.md) for Infromation Retrieval +e.g.: [webpage/content/tasks/taskQA.md](https://raw.githubusercontent.com/CLARIN-PL/embeddings/main/webpage/content/tasks/taskQA.md) for Question Answering | Keys to be filled | Example | | -- | -- | -| **[Task Name with CamelCase]** | InformationRetrieval | -| **[Task Name]** | Information Retrieval | -| **[Task Description]** | Information Retrieval (IR) is the process of locating and retrieving relevant data or documents from a large dataset based on user queries, often used in search engines | +| **[Task Name with CamelCase]** | QuestionAnswering | +| **[Task Name]** | Question Answering | +| **[Task Description]** | Question Answering (QA) is the task of automatically returning an answer to a natural-language question, given a passage or knowledge source. | Content to be filled replace **squared [] braces** : ```html @@ -265,6 +238,5 @@ const taskTypes = [ "Abusive Clauses Detection", "Dialogue Acts Classification", "Question Answering" - "Information Retrieval" ]; ``` \ No newline at end of file diff --git a/webpage/content/tasks/taskIR.md b/webpage/content/tasks/taskIR.md deleted file mode 100644 index 5789951d..00000000 --- a/webpage/content/tasks/taskIR.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -url: "/tasks/InformationRetrieval" -type: docs -geekdocNav: false -geekdocBreadcrumb: false ---- - -{{< pageHeader >}} -{{< info taskname="Information Retrieval" taskdesc="Information Retrieval (IR) is the process of locating and retrieving relevant data or documents from a large dataset based on user queries, often used in search engines">}} -{{< averageResults tasktype="Information Retrieval" >}} -{{< results type="Information Retrieval" >}} \ No newline at end of file diff --git a/webpage/data/results/msmarco_bm_25.json b/webpage/data/results/msmarco_bm_25.json deleted file mode 100644 index f114d5d4..00000000 --- a/webpage/data/results/msmarco_bm_25.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "submission_name": "msmarco_bm_25", - "dataset_name": "MSMARCO", - "dataset_version": "0.0.0", - "embedding_name": "BM25", - "hparams": {}, - "packages": [], - "config": null, - "leaderboard_task_name": "Information Retrieval", - "metrics": [ - { - "NDCG@10": 31.50, - "MRR@10": 56.36 - } - ], - "metrics_avg": { - "NDCG@10": 31.50, - "MRR@10": 56.36 - }, - "metrics_std": { - "NDCG@10": 0.0, - "MRR@10": 0.0 - }, - "averaged_over": 1 -} \ No newline at end of file diff --git a/webpage/data/results/msmarco_ict.json b/webpage/data/results/msmarco_ict.json deleted file mode 100644 index f1b5915e..00000000 --- a/webpage/data/results/msmarco_ict.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "submission_name": "msmarco_ict", - "dataset_name": "MSMARCO", - "dataset_version": "0.0.0", - "embedding_name": "ICT", - "hparams": {}, - "packages": [], - "config": null, - "leaderboard_task_name": "Information Retrieval", - "metrics": [ - { - "NDCG@10": 29.02, - "MRR@10": 43.91 - } - ], - "metrics_avg": { - "NDCG@10": 29.02, - "MRR@10": 43.91 - }, - "metrics_std": { - "NDCG@10": 0.0, - "MRR@10": 0.0 - }, - "averaged_over": 1 -} \ No newline at end of file diff --git a/webpage/layouts/shortcodes/averageResults.html b/webpage/layouts/shortcodes/averageResults.html index b341b841..b5bf019e 100644 --- a/webpage/layouts/shortcodes/averageResults.html +++ b/webpage/layouts/shortcodes/averageResults.html @@ -9,7 +9,7 @@ {{ $metricsToIgnore := slice "classes" "f1_weighted" "recall_weighted" "precision_weighted" }} {{ range $file, $result := $.Site.Data.results }} {{if or (eq $tasktype "Tasks") (eq (index $result "leaderboard_task_name") $tasktype)}} - {{if (not (or (eq (index $result "leaderboard_task_name") "Information Retrieval") (eq (index $result "leaderboard_task_name") "Question Answering")))}} + {{if (not (eq (index $result "leaderboard_task_name") "Question Answering"))}} {{ range $k, $v := .metrics_avg }} {{if (not (in $metricsToIgnore $k)) }} {{ $metrics = uniq ($metrics | append $k) }} @@ -36,7 +36,7 @@
Model ranking

Average model performance across all tasks.

{{else}} - {{if (not (or (eq $tasktype "Information Retrieval") (eq $tasktype "Question Answering")))}} + {{if (not (eq $tasktype "Question Answering"))}}
Model's task ranking

Average model performance across task datasets.

{{end}} @@ -44,7 +44,7 @@ {{end}} - {{if (not (or (eq $tasktype "Information Retrieval") (eq $tasktype "Question Answering")))}} + {{if (not (eq $tasktype "Question Answering"))}}