galleries/tasks/6_evals.json at main · transformerlab/galleries · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
[
  {
    "title": "Evaluation Task Example",
    "description": "A sample evaluation task that runs an evaluation script and creates reports.",
    "github_repo_url": "https://github.com/transformerlab/transformerlab-examples",
    "github_repo_dir": "demo-eval-task",
    "metadata": {
      "category": "eval",
      "modality": "text",
      "framework": ["huggingface"]
    },
    "supportedAccelerators": {
      "NVIDIA": {
        "resources": {
          "cpus": "2",
          "memory": "4"
        }
      },
      "AMD": {
        "resources": {
          "cpus": "2",
          "memory": "4"
        }
      },
      "AppleSilicon": {
        "resources": {
          "cpus": "2",
          "memory": "4"
        }
      },
      "cpu": {
        "resources": {
          "cpus": "2",
          "memory": "4"
        }
      }
    }
  },
  {
    "title": "Basic Evaluation Metrics",
    "description": "Evaluating outputs of LLMs using basic defined metrics including regex patterns, word counts, and custom Python code.",
    "github_repo_url": "https://github.com/transformerlab/transformerlab-examples",
    "github_repo_dir": "basic-evals-metrics",
    "metadata": {
      "category": "eval",
      "modality": "text",
      "framework": [""]
    },
    "supportedAccelerators": {
      "NVIDIA": {
        "resources": {
          "accelerators": "RTX3090:1"
        }
      },
      "AMD": {
        "resources": {
          "accelerators": "7900XTX:1"
        }
      }
    }
  },
  {
    "title": "EleutherAI LM Evaluation Harness",
    "description": "EleutherAI LM Evaluation Harness for evaluating language models on various benchmarks.",
    "github_repo_url": "https://github.com/transformerlab/transformerlab-examples",
    "github_repo_dir": "eleutherai-lm-evaluation-harness",
    "metadata": {
      "category": "eval",
      "modality": "text",
      "framework": ["eleutherai"]
    },
    "supportedAccelerators": {
      "NVIDIA": {
        "resources": {
          "accelerators": "RTX3090:1"
        }
      },
      "AMD": {
        "resources": {
          "accelerators": "7900XTX:1"
        }
      }
    }
  },
  {
    "title": "Objective Metrics Evaluation",
    "description": "Evaluating outputs of LLMs using objective metrics powered by DeepEval Framework. Supports metrics like Rouge, BLEU, Exact Match, Quasi Exact Match, Quasi Contains, and BERT Score.",
    "github_repo_url": "https://github.com/transformerlab/transformerlab-examples",
    "github_repo_dir": "objective-eval",
    "metadata": {
      "category": "eval",
      "modality": "text",
      "framework": ["deepeval"]
    },
    "supportedAccelerators": {
      "NVIDIA": {
        "resources": {
          "accelerators": "RTX3090:1"
        }
      },
      "AMD": {
        "resources": {
          "accelerators": "7900XTX:1"
        }
      }
    }
  }
]