ナード戦隊データマン

機械学習と自然言語処理についてのブログ

エンティティ名と定義文からタイプ予測

keras-self-attention1 は、self-attentionを簡単に使うためのライブラリです。今回は、このライブラリを使うことによって、エンティティタイプ予測の問題をより汎用的な特徴量から学習させられることを確かめます。

事前準備

pip install keras-self-attention

訓練部分のコード

import json
import sqlite3
from random import choice
 
import numpy as np
import sentencepiece as spm
from keras import optimizers
from keras.callbacks import ModelCheckpoint
from keras.layers import Dense, Input, Embedding, Dropout, concatenate, Flatten
from keras.models import Model
from keras.preprocessing.sequence import pad_sequences
from keras_self_attention import SeqSelfAttention
 
sp = spm.SentencePieceProcessor()
sp.load("./spmodel/en.wiki.bpe.vs10000.model")
 
 
def build_model(num_labels,
                maxlen1=128,
                maxlen2=128,
                max_features=10001,
                dim1=50,
                dim2=50):
    inputs = Input(shape=(maxlen1, ))
    inputs2 = Input(shape=(maxlen2, ))
    emb1 = Embedding(max_features, dim1, input_length=maxlen1)(inputs)
    emb2 = Embedding(max_features, dim2, input_length=maxlen2)(inputs2)
    tmp = concatenate([emb1, emb2])
    att = SeqSelfAttention(attention_activation='sigmoid')(tmp)
    tmp2 = Flatten()(emb1)
    tmp3 = Flatten()(att)
    tmp = concatenate([tmp2, tmp3])
    tmp = Dense(1024, activation="linear")(tmp)
    out = Dropout(0.5)(tmp)
    output = Dense(units=num_labels, activation='softmax')(out)
    model = Model([inputs, inputs2], output)
    opt = optimizers.Adam(lr=0.001, clipvalue=0.5)
    model.compile(loss='categorical_crossentropy',
                  optimizer=opt,
                  metrics=['acc'])
    return model
 
 
def preprocessing(texts, sp=sp, maxlen=50):
    return pad_sequences([sp.EncodeAsIds(text) for text in texts],
                         maxlen=maxlen)
 
 
def generate_data(data, labels, batch_size=1000):
    X0 = []
    X1 = []
    y = []
    while True:
        for i, d in enumerate(data):
            X0.append(d["entity"].replace("_", " "))
            X1.append(d["definition"])
            label = np.zeros(len(labels.keys()))
            label[int(labels[d["type"]])] = 1.0
            y.append(label)
            if len(X0) > 1000:
                yield [preprocessing(X0, maxlen=128), preprocessing(X1, maxlen=128)], np.array(y)
                X0 = []
                X1 = []
                y = []
 
def generate_test_data(test, labels):
    X0 = []
    X1 = []
    y = []
    for i, d in enumerate(test):
        X0.append(d["entity"].replace("_", " "))
        X1.append(d["definition"])
        y.append(int(labels[d["type"]]))
    return [preprocessing(X0, maxlen=128), preprocessing(X1, maxlen=128)], np.array(y)
 
 
if __name__ == "__main__":
    from sklearn.utils import shuffle
    from sklearn.metrics import classification_report
    with open("./labels.json") as f:
        key2i = json.load(f)[1]
 
    with open("./wiki_definition.json") as f:
        data = shuffle(json.load(f))
 
    size = int(len(data) * 0.8)
    train, test = data[:size], data[size:]
    size = int(len(test) * 0.8)
    test, val = test[:size], test[size:]
 
    model = build_model(len(key2i.keys()))
    callbacks = [
        ModelCheckpoint("model.h5",
                        save_best_only=False,
                        monitor="val_loss",
                        mode="min"),
        ModelCheckpoint("model_best.h5",
                        save_best_only=True,
                        monitor="val_loss",
                        mode="min")
    ]
    model.fit_generator(generate_data(train, key2i),
                        validation_data=generate_data(val, key2i),
                        validation_steps=10,
                        steps_per_epoch=1000,
                        epochs=3,
                        verbose=1,
                        callbacks=callbacks)
 
    X_test, y_test = generate_test_data(test, key2i)
 
    y_pred = [np.argmax(x) for x in model.predict(X_test)]
 
    with open("eval.txt", "w") as f:
        f.write(classification_report(y_test, y_pred))

Note: labels.jsonには、ラベル名とラベルIDの対応が入っています。

Note: wiki_definition.jsonには、定義文、エンティティ名, エンティティタイプが格納されています。

精度

スマホなど、ブラウザの横幅が狭い場合は、以下の精度を横にスクロールさせて見てください。

                                             precision recall    f1-score  support   

AcademicConference                           0.71      0.33      0.45      15        
AcademicJournal                              0.92      0.95      0.94      1075      
Actor                                        0.49      0.16      0.24      630       
AdministrativeRegion                         0.84      0.81      0.82      3307      
AdultActor                                   0.76      0.61      0.68      171       
Aircraft                                     0.92      0.96      0.94      1726      
Airline                                      0.96      0.95      0.96      582       
Airport                                      0.97      0.94      0.96      2177      
Album                                        0.96      0.96      0.96      19501     
AmateurBoxer                                 0.29      0.08      0.12      75        
Ambassador                                   0.28      0.36      0.31      83        
AmericanFootballLeague                       0.00      0.00      0.00      10        
AmericanFootballPlayer                       0.80      0.90      0.85      3070      
AmericanFootballTeam                         0.00      0.00      0.00      5         
Amphibian                                    0.91      0.95      0.93      563       
AmusementParkAttraction                      0.76      0.55      0.64      64        
AnatomicalStructure                          0.56      0.64      0.60      268       
Animal                                       0.73      0.83      0.77      1096      
AnimangaCharacter                            0.50      0.05      0.08      22        
Anime                                        0.41      0.10      0.16      161       
Arachnid                                     0.94      0.84      0.89      632       
Archaea                                      0.71      0.56      0.63      27        
Architect                                    0.58      0.34      0.43      457       
ArchitecturalStructure                       0.33      0.05      0.08      43        
Artery                                       1.00      0.83      0.91      52        
ArtificialSatellite                          0.89      0.94      0.91      371       
Artist                                       0.47      0.38      0.42      2267      
ArtistDiscography                            0.98      0.99      0.98      560       
Artwork                                      0.82      0.81      0.82      760       
Asteroid                                     0.00      0.00      0.00      2         
Astronaut                                    0.77      0.79      0.78      104       
Athlete                                      0.81      0.86      0.84      5763      
AustralianFootballLeague                     0.00      0.00      0.00      1         
AustralianFootballTeam                       0.72      0.88      0.79      72        
AustralianRulesFootballPlayer                0.98      0.99      0.98      1959      
AutoRacingLeague                             0.00      0.00      0.00      1         
Automobile                                   0.92      0.86      0.89      887       
AutomobileEngine                             0.87      0.76      0.81      54        
Award                                        0.82      0.93      0.87      788       
Bacteria                                     0.45      0.37      0.41      145       
BadmintonPlayer                              0.96      0.88      0.92      241       
Band                                         0.86      0.79      0.83      5259      
Bank                                         0.72      0.41      0.52      425       
Baronet                                      0.68      0.81      0.74      118       
BaseballLeague                               0.58      0.88      0.70      48        
BaseballPlayer                               0.96      0.98      0.97      3303      
BaseballSeason                               0.96      0.42      0.58      53        
BaseballTeam                                 0.00      0.00      0.00      2         
BasketballLeague                             0.88      0.62      0.73      69        
BasketballPlayer                             0.92      0.94      0.93      1930      
BasketballTeam                               0.87      0.82      0.85      271       
Bay                                          0.00      0.00      0.00      1         
BeachVolleyballPlayer                        0.55      0.43      0.48      28        
BeautyQueen                                  0.87      0.76      0.81      339       
Beverage                                     0.79      0.56      0.66      130       
BiologicalDatabase                           0.66      0.67      0.67      46        
Bird                                         0.98      0.92      0.95      2012      
BodyOfWater                                  0.68      0.45      0.54      195       
Bodybuilder                                  0.90      0.43      0.58      44        
Bone                                         0.65      0.47      0.55      55        
Book                                         0.85      0.90      0.87      5439      
Boxer                                        0.85      0.86      0.85      642       
Brain                                        0.78      0.77      0.77      82        
Brewery                                      0.18      0.07      0.10      43        
Bridge                                       0.85      0.92      0.88      694       
BroadcastNetwork                             0.34      0.28      0.31      197       
Building                                     0.67      0.78      0.72      7466      
BusCompany                                   0.87      0.74      0.80      223       
BusinessPerson                               0.00      0.00      0.00      109       
CanadianFootballLeague                       0.00      0.00      0.00      1         
CanadianFootballTeam                         0.56      0.29      0.38      31        
Canal                                        0.84      0.65      0.73      63        
Canoeist                                     0.90      0.89      0.89      70        
Cardinal                                     0.57      0.35      0.43      117       
Castle                                       0.60      0.72      0.65      232       
Cave                                         0.87      0.67      0.76      89        
Chancellor                                   0.00      0.00      0.00      16        
Cheese                                       0.73      0.71      0.72      42        
Chef                                         0.61      0.50      0.55      90        
ChemicalCompound                             0.86      0.90      0.88      1550      
ChessPlayer                                  0.85      0.80      0.82      226       
ChristianBishop                              0.75      0.77      0.76      1413      
City                                         0.70      0.76      0.73      3115      
ClassicalMusicArtist                         0.00      0.00      0.00      60        
ClassicalMusicComposition                    0.55      0.36      0.44      99        
Cleric                                       0.71      0.48      0.57      414       
ClubMoss                                     0.00      0.00      0.00      17        
College                                      0.86      0.86      0.86      14        
CollegeCoach                                 0.83      0.82      0.83      1053      
Colour                                       0.75      0.63      0.69      38        
Comedian                                     0.49      0.23      0.32      194       
ComedyGroup                                  0.00      0.00      0.00      7         
Comic                                        0.75      0.79      0.77      344       
ComicStrip                                   0.88      0.46      0.61      65        
ComicsCharacter                              0.89      0.84      0.87      620       
ComicsCreator                                0.71      0.55      0.62      426       
Company                                      0.76      0.84      0.80      7920      
ConcentrationCamp                            0.33      0.08      0.13      12        
Congressman                                  0.42      0.47      0.45      493       
Conifer                                      0.86      0.75      0.80      110       
Constellation                                0.90      0.60      0.72      15        
Continent                                    0.00      0.00      0.00      5         
Convention                                   0.58      0.59      0.59      327       
Country                                      0.70      0.52      0.60      475       
Crater                                       0.91      0.81      0.86      103       
CricketGround                                0.90      0.70      0.79      37        
CricketLeague                                0.00      0.00      0.00      1         
CricketTeam                                  0.91      0.83      0.87      113       
Cricketer                                    0.97      0.97      0.97      2926      
Criminal                                     0.66      0.40      0.50      380       
Crustacean                                   0.87      0.83      0.85      433       
CultivatedVariety                            0.97      0.91      0.94      242       
Curler                                       0.90      0.93      0.91      137       
Currency                                     0.92      0.77      0.84      57        
Cycad                                        0.83      0.54      0.65      28        
CyclingRace                                  0.85      0.94      0.89      115       
CyclingTeam                                  0.97      0.74      0.84      47        
Cyclist                                      0.94      0.96      0.95      1745      
Dam                                          0.91      0.86      0.89      516       
DartsPlayer                                  0.97      0.86      0.91      101       
Device                                       0.91      0.83      0.87      196       
Diocese                                      0.98      0.97      0.97      526       
Disease                                      0.95      0.82      0.88      882       
Drug                                         0.88      0.79      0.84      936       
Earthquake                                   0.99      0.98      0.98      129       
Economist                                    0.46      0.20      0.28      188       
EducationalInstitution                       0.84      0.41      0.55      75        
Election                                     1.00      0.71      0.83      17        
Embryology                                   0.39      0.30      0.34      30        
Engineer                                     0.46      0.21      0.29      115       
Entomologist                                 0.56      0.26      0.36      73        
Enzyme                                       0.96      0.96      0.96      786       
EthnicGroup                                  0.87      0.79      0.83      773       
Eukaryote                                    0.62      0.42      0.50      208       
EurovisionSongContestEntry                   0.84      0.86      0.85      179       
Event                                        0.75      0.69      0.72      655       
Fashion                                      0.33      0.02      0.04      50        
FashionDesigner                              0.66      0.35      0.46      130       
Fern                                         0.82      0.80      0.81      165       
FictionalCharacter                           0.77      0.46      0.57      640       
FieldHockeyLeague                            0.00      0.00      0.00      5         
FigureSkater                                 0.97      0.94      0.95      528       
Film                                         0.92      0.93      0.93      16836     
FilmFestival                                 0.91      0.86      0.88      143       
Fish                                         0.95      0.93      0.94      2870      
FloweringPlant                               0.00      0.00      0.00      26        
Food                                         0.89      0.85      0.87      744       
FootballLeagueSeason                         0.93      0.96      0.95      1501      
FootballMatch                                0.96      0.89      0.92      486       
FormulaOneRacer                              0.69      0.47      0.56      120       
FormulaOneTeam                               0.92      0.58      0.71      19        
Fungus                                       0.94      0.96      0.95      1742      
GaelicGamesPlayer                            0.95      0.96      0.96      568       
Galaxy                                       0.94      0.95      0.95      157       
Game                                         0.82      0.67      0.74      228       
Garden                                       0.81      0.42      0.55      50        
Ginkgo                                       0.00      0.00      0.00      2         
GivenName                                    0.90      0.90      0.90      621       
Glacier                                      0.92      0.83      0.87      119       
Gnetophytes                                  0.00      0.00      0.00      3         
GolfCourse                                   0.86      0.74      0.80      50        
GolfLeague                                   0.00      0.00      0.00      3         
GolfPlayer                                   0.97      0.94      0.96      538       
GolfTournament                               0.93      0.97      0.95      269       
GovernmentAgency                             0.72      0.78      0.75      879       
Governor                                     0.43      0.18      0.26      420       
GrandPrix                                    0.97      0.98      0.98      229       
Grape                                        0.93      0.83      0.88      69        
GreenAlga                                    0.48      0.75      0.58      67        
GridironFootballPlayer                       0.69      0.45      0.54      1090      
Guitarist                                    0.00      0.00      0.00      23        
Gymnast                                      0.91      0.80      0.85      333       
HandballLeague                               0.00      0.00      0.00      3         
HandballPlayer                               0.94      0.96      0.95      366       
HandballTeam                                 0.90      0.78      0.84      60        
Historian                                    0.40      0.03      0.06      124       
HistoricBuilding                             0.79      0.81      0.80      1373      
HistoricPlace                                0.66      0.42      0.51      3697      
HockeyTeam                                   0.93      0.90      0.92      374       
Holiday                                      0.75      0.59      0.66      152       
HollywoodCartoon                             0.82      0.83      0.83      241       
HorseRace                                    0.97      0.98      0.98      378       
HorseRider                                   0.76      0.69      0.72      87        
HorseTrainer                                 0.83      0.39      0.54      38        
Hospital                                     0.84      0.90      0.87      474       
Hotel                                        0.68      0.49      0.57      196       
IceHockeyLeague                              0.77      0.82      0.79      44        
IceHockeyPlayer                              0.95      0.97      0.96      2344      
InformationAppliance                         0.80      0.45      0.57      200       
InlineHockeyLeague                           0.00      0.00      0.00      1         
Insect                                       0.98      0.99      0.98      21033     
Island                                       0.76      0.81      0.78      929       
Jockey                                       0.73      0.74      0.73      91        
Journalist                                   0.30      0.12      0.17      230       
Judge                                        0.52      0.30      0.38      430       
LacrosseLeague                               1.00      0.50      0.67      8         
LacrossePlayer                               0.77      0.84      0.81      69        
Lake                                         0.91      0.90      0.91      1516      
Language                                     0.97      0.94      0.95      1191      
LaunchPad                                    1.00      0.86      0.92      14        
LawFirm                                      0.87      0.49      0.62      68        
Legislature                                  0.79      0.85      0.82      287       
Library                                      0.65      0.82      0.73      158       
Ligament                                     0.96      0.79      0.87      29        
Lighthouse                                   0.95      0.87      0.91      287       
Locomotive                                   0.87      0.95      0.91      515       
Lymph                                        0.83      1.00      0.91      10        
Magazine                                     0.90      0.67      0.77      809       
Mammal                                       0.89      0.89      0.89      1260      
Manga                                        0.73      0.55      0.63      491       
MartialArtist                                0.92      0.70      0.79      513       
Mayor                                        0.42      0.41      0.41      266       
Medician                                     0.17      0.01      0.02      76        
MemberOfParliament                           0.52      0.61      0.56      1191      
MilitaryConflict                             0.92      0.92      0.92      2103      
MilitaryPerson                               0.84      0.79      0.81      4300      
MilitaryStructure                            0.64      0.52      0.57      665       
MilitaryUnit                                 0.94      0.93      0.94      2694      
Mineral                                      0.94      0.93      0.93      224       
MixedMartialArtsEvent                        0.95      0.91      0.93      116       
Model                                        0.62      0.46      0.53      252       
Mollusca                                     0.98      0.98      0.98      4271      
Monarch                                      0.49      0.36      0.42      367       
Monument                                     0.56      0.22      0.32      86        
Moss                                         0.85      0.64      0.73      78        
Motorcycle                                   0.84      0.86      0.85      153       
MotorcycleRacingLeague                       0.00      0.00      0.00      4         
MotorcycleRider                              0.86      0.93      0.89      201       
MotorsportSeason                             0.94      0.93      0.94      503       
Mountain                                     0.91      0.88      0.89      2631      
MountainPass                                 0.93      0.88      0.90      160       
MountainRange                                0.91      0.78      0.84      385       
Murderer                                     0.00      0.00      0.00      17        
Muscle                                       0.92      0.77      0.84      44        
Museum                                       0.83      0.68      0.75      874       
MusicFestival                                0.73      0.34      0.47      64        
MusicGenre                                   0.83      0.49      0.62      172       
Musical                                      0.76      0.83      0.79      209       
MusicalArtist                                0.70      0.73      0.71      7874      
MusicalWork                                  0.50      0.14      0.22      43        
MythologicalFigure                           0.73      0.57      0.64      109       
NCAATeamSeason                               1.00      1.00      1.00      2668      
NascarDriver                                 0.64      0.84      0.72      146       
NationalCollegiateAthleticAssociationAthlete 0.00      0.00      0.00      23        
NationalFootballLeagueSeason                 0.96      0.97      0.96      560       
Nerve                                        0.87      0.86      0.86      56        
NetballPlayer                                0.91      0.69      0.78      42        
Newspaper                                    0.85      0.87      0.86      1019      
Noble                                        0.60      0.48      0.53      799       
Non-ProfitOrganisation                       0.30      0.01      0.02      303       
Novel                                        0.00      0.00      0.00      4         
OfficeHolder                                 0.61      0.70      0.65      10451     
OlympicEvent                                 0.98      0.97      0.98      726       
OlympicResult                                0.80      0.99      0.89      113       
Olympics                                     0.75      1.00      0.86      6         
Organisation                                 0.67      0.60      0.63      2759      
Painter                                      0.68      0.35      0.46      402       
Park                                         0.64      0.48      0.55      608       
Person                                       0.59      0.67      0.63      28250     
Philosopher                                  0.41      0.17      0.24      286       
Photographer                                 0.75      0.04      0.08      73        
Place                                        0.68      0.68      0.68      1071      
Planet                                       0.98      0.95      0.96      537       
Plant                                        0.97      0.96      0.96      8292      
Play                                         0.75      0.62      0.68      288       
PlayboyPlaymate                              0.82      0.73      0.78      45        
Poem                                         0.57      0.53      0.55      57        
Poet                                         0.34      0.21      0.26      56        
PokerPlayer                                  0.96      0.94      0.95      130       
PoliticalParty                               0.93      0.84      0.88      1178      
Politician                                   0.75      0.25      0.38      2808      
PoloLeague                                   0.00      0.00      0.00      5         
Pope                                         0.94      0.88      0.91      68        
PowerStation                                 0.83      0.95      0.89      326       
Presenter                                    0.00      0.00      0.00      16        
President                                    0.54      0.14      0.22      321       
PrimeMinister                                0.33      0.06      0.11      203       
Prison                                       0.89      0.89      0.89      199       
ProgrammingLanguage                          0.73      0.55      0.63      148       
ProtectedArea                                0.80      0.85      0.82      1404      
Protein                                      0.81      0.75      0.78      356       
PublicTransitSystem                          0.49      0.68      0.57      252       
Publisher                                    0.81      0.43      0.56      250       
RaceHorse                                    0.97      0.97      0.97      627       
Racecourse                                   0.81      0.57      0.67      51        
RacingDriver                                 0.74      0.73      0.74      388       
RadioHost                                    0.17      0.32      0.22      56        
RadioProgram                                 0.71      0.62      0.66      189       
RadioStation                                 0.96      0.97      0.96      3086      
RailwayLine                                  0.90      0.77      0.83      526       
RailwayStation                               0.93      0.60      0.73      235       
RailwayTunnel                                0.75      0.75      0.75      32        
RecordLabel                                  0.88      0.85      0.86      506       
Religious                                    0.43      0.14      0.21      169       
ReligiousBuilding                            0.59      0.67      0.62      675       
Reptile                                      0.90      0.90      0.90      742       
ResearchProject                              0.00      0.00      0.00      2         
Restaurant                                   0.79      0.63      0.70      175       
River                                        0.96      0.98      0.97      4331      
Road                                         0.95      0.98      0.96      3216      
RoadJunction                                 0.86      0.64      0.73      28        
RoadTunnel                                   0.65      0.84      0.73      43        
Rocket                                       0.78      0.45      0.57      40        
RollerCoaster                                0.87      0.86      0.87      109       
Rower                                        0.75      0.28      0.41      43        
Royalty                                      0.71      0.67      0.69      1580      
RugbyClub                                    0.85      0.89      0.87      366       
RugbyLeague                                  0.77      0.65      0.70      62        
RugbyPlayer                                  0.95      0.97      0.96      2539      
Saint                                        0.75      0.72      0.73      602       
School                                       0.91      0.96      0.93      4936      
Scientist                                    0.51      0.53      0.52      3674      
ScreenWriter                                 0.36      0.03      0.06      118       
Sea                                          0.50      0.14      0.22      7         
Senator                                      0.21      0.16      0.18      115       
Settlement                                   0.88      0.91      0.90      37384     
Ship                                         0.96      0.97      0.97      4584      
ShoppingMall                                 0.86      0.89      0.88      381       
Single                                       0.88      0.93      0.90      7687      
SiteOfSpecialScientificInterest              0.79      0.88      0.83      170       
Skater                                       0.65      0.80      0.72      80        
SkiArea                                      0.85      0.70      0.77      92        
Skier                                        0.77      0.84      0.81      413       
Skyscraper                                   0.00      0.00      0.00      1         
SnookerChamp                                 0.00      0.00      0.00      4         
SnookerPlayer                                0.83      0.95      0.89      61        
SoapCharacter                                0.85      0.84      0.85      418       
SoccerClub                                   0.97      0.94      0.95      3233      
SoccerClubSeason                             0.94      0.99      0.96      1460      
SoccerLeague                                 0.81      0.77      0.79      269       
SoccerManager                                0.82      0.64      0.72      2965      
SoccerPlayer                                 0.93      0.97      0.95      17824     
SoccerTournament                             0.84      0.92      0.88      1079      
SoftballLeague                               0.00      0.00      0.00      2         
Software                                     0.78      0.84      0.81      1716      
SolarEclipse                                 0.98      0.98      0.98      61        
Song                                         0.56      0.29      0.38      941       
SpaceShuttle                                 0.00      0.00      0.00      2         
SpaceStation                                 0.00      0.00      0.00      9         
Species                                      0.79      0.81      0.80      940       
SpeedwayLeague                               0.00      0.00      0.00      3         
SpeedwayRider                                0.95      0.97      0.96      108       
SpeedwayTeam                                 1.00      0.67      0.80      9         
Sport                                        0.38      0.25      0.30      40        
SportsEvent                                  0.84      0.80      0.82      172       
SportsLeague                                 0.31      0.28      0.29      65        
SportsTeam                                   0.63      0.69      0.66      377       
SportsTeamMember                             0.93      0.59      0.72      90        
SquashPlayer                                 0.97      0.94      0.95      65        
Stadium                                      0.62      0.30      0.40      932       
Star                                         0.96      0.96      0.96      469       
Station                                      0.95      0.99      0.97      4268      
Stream                                       0.00      0.00      0.00      7         
SumoWrestler                                 0.97      0.87      0.92      76        
SupremeCourtOfTheUnitedStatesCase            0.97      0.97      0.97      437       
Surname                                      0.14      0.01      0.02      74        
Swimmer                                      0.88      0.82      0.85      919       
TableTennisPlayer                            0.82      0.68      0.74      75        
TelevisionEpisode                            0.93      0.92      0.92      1277      
TelevisionHost                               0.00      0.00      0.00      10        
TelevisionSeason                             0.94      0.88      0.91      569       
TelevisionShow                               0.86      0.83      0.84      5644      
TelevisionStation                            0.91      0.84      0.87      1163      
TennisLeague                                 0.00      0.00      0.00      1         
TennisPlayer                                 0.92      0.97      0.94      810       
TennisTournament                             0.92      0.94      0.93      188       
Theatre                                      0.30      0.24      0.27      115       
Town                                         0.83      0.71      0.76      6528      
TradeUnion                                   0.89      0.73      0.81      248       
Train                                        0.91      0.76      0.83      237       
Tunnel                                       0.00      0.00      0.00      15        
University                                   0.88      0.86      0.87      2900      
Valley                                       0.14      0.07      0.09      15        
Vein                                         0.92      0.94      0.93      35        
Venue                                        0.41      0.67      0.51      889       
VideoGame                                    0.90      0.93      0.92      3038      
Village                                      0.96      0.95      0.95      26308     
VoiceActor                                   0.00      0.00      0.00      28        
Volcano                                      0.68      0.51      0.58      106       
VolleyballCoach                              0.08      0.12      0.10      8         
VolleyballLeague                             0.71      0.38      0.50      13        
VolleyballPlayer                             0.94      0.89      0.91      601       
WaterRide                                    0.00      0.00      0.00      10        
WaterwayTunnel                               1.00      0.50      0.67      2         
Weapon                                       0.90      0.78      0.84      814       
Website                                      0.73      0.29      0.42      548       
WineRegion                                   0.94      0.77      0.85      65        
Winery                                       0.82      0.73      0.77      51        
WomensTennisAssociationTournament            0.91      0.93      0.92      88        
WorldHeritageSite                            0.57      0.07      0.13      107       
Wrestler                                     0.89      0.73      0.80      524       
WrestlingEvent                               0.95      0.91      0.93      173       
Writer                                       0.50      0.33      0.40      4675      
WrittenWork                                  0.66      0.81      0.73      235       
Year                                         0.99      0.97      0.98      247       
YearInSpaceflight                            1.00      0.86      0.92      7         
owl#Thing                                    0.52      0.56      0.54      42119     

micro avg                                    0.81      0.81      0.81      500412    
macro avg                                    0.69      0.61      0.64      500412    
weighted avg                                 0.81      0.81      0.81      500412