ナード戦隊データマン

データサイエンスを用いて悪と戦うぞ

Wikipediaのタイプ予測

DBPedia Instance Typesは、DBPediaのオントロジー情報によって各記事を分類したものです。今回は、Wikipediaの記事をこのDBPedia Instance Typesで分類してみます。

特徴量

  1. Wikipedia記事の定義文の単語の平均ベクトル。
  2. Wikipedia記事のカテゴリー。

実行の流れ

  1. DBPediaのType情報を持っているWikipedia記事を取得し、タイプと記事名を対応付けて保存。
  2. 記事名に対する定義文を抽出。
  3. 記事名に対するカテゴリーを抽出。
  4. 事前訓練済みword2vecを使って定義文の平均ベクトルを生成。
  5. 記事のもつ全カテゴリー名をスペース結合してからBoWする。
  6. 平均ベクトルとBoWを水平結合。
  7. 水平結合したスパース行列をSparseRandomProjectionで次元削減。
  8. タイプ情報をOneHot化してラベルとして使う。
  9. 次元削減された特徴量とラベルを使ってKerasモデルを訓練。
  10. 精度を出す。
  11. 最終的に、このモデルを使って未知のWikipedia記事を、定義文とカテゴリーからInstance Typesで分類できる。

訓練部分のコード

import pickle
import numpy as np
from keras import optimizers
from keras.callbacks import ModelCheckpoint
from keras.layers import Dense, Input, Dropout
from keras.models import Model
from scipy.sparse import coo_matrix, hstack
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from sklearn.random_projection import SparseRandomProjection
from tqdm import tqdm


def build_model(num_labels, inputlen):
    inp = Input(shape=(inputlen, ))
    tmp = Dense(1024, activation="relu", kernel_initializer="he_normal")(inp)
    tmp = Dropout(0.5)(tmp)
    tmp = Dense(1042, activation="relu", kernel_initializer="he_normal")(tmp)
    tmp = Dropout(0.5)(tmp)
    out = Dense(units=num_labels, activation='softmax')(tmp)
    model = Model(inp, out)
    opt = optimizers.Adam(lr=0.001, clipvalue=0.5)
    model.compile(loss='categorical_crossentropy',
                  optimizer=opt,
                  metrics=['acc'])
    return model


def gen_data(data, page2cat, kv, labels):
    X = []
    X_cat = []
    y = []
    for d in tqdm(data):
        tmp = []
        tmp_cat = ""
        for cat in page2cat[d["entity"]]:
            tmp_cat += cat.lower() + " "
        tmp_cat = tmp_cat.replace("_", " ")

        for word in d["definition"].replace(".", " ").replace(",",
                                                              " ").split():
            try:
                x = kv.wv[word]
            except Exception:
                continue
            tmp.append(x)
        if tmp:
            X.append(np.mean(tmp, axis=0))
            X_cat.append(tmp_cat)
            y.append(int(labels[d["type"]]))
    vect = CountVectorizer().fit(X_cat)
    X_cat = vect.transform(X_cat)
    X = coo_matrix(X)
    X = hstack([X, X_cat])
    return X, y, vect


def y_fix(y, ylen):
    out = []
    for target in y:
        tmp = np.zeros(ylen)
        tmp[int(target)] = 1.0
        out.append(tmp)
    return out


def generate_data(X, y, batch_size=1000):
    y = np.array(y)
    print(X.shape)
    print(y.shape)
    while True:
        for i in range(int(X.shape[0]/1000)):
            yield X[i*batch_size:(i+1)*batch_size], y[i*batch_size:(i+1)*batch_size]
       


if __name__ == "__main__":
    #print("data loading...")
    #with open("./wiki_definition.json") as f:
    #data = json.load(f)

    #with open("../scripts/labels.json") as f:
    #    tmp = json.load(f)
    #    labels = tmp[1]

    #data = shuffle(data)
    #kv = KeyedVectors.load("./enwiki_model/word2vec.model", mmap="r")
    #with open("./categories.pkl", "rb") as f:
    #    page2cat = pickle.load(f)
    #print("done loading")

    #print("generating data...")
    #X, y, vect = gen_data(data, page2cat, kv, labels)
    #print("done generating")

    #print("saveing data...")
    #with open("data.pkl", "wb") as f:
    #    pickle.dump((X, y, vect), f, protocol=4)
    #print("done saving")

    print("loading")
    with open("data.pkl", "rb") as f:
        X, y, _ = pickle.load(f)
    print("done loading")

    y_len = np.unique(y).shape[0] + 1
    print(y_len)
    y = y_fix(y, y_len)

    print("projection")
    projection = SparseRandomProjection(n_components=1024).fit(X)
    X = projection.transform(X)
    with open("proj.pkl", "wb") as f:
        pickle.dump(projection, f)
    print(X.shape)
    X = X.toarray()
    print("done projection")

    print("training...")
    X_train, X_test, y_train, y_test = train_test_split(X, y)
    X_test, X_val, y_test, y_val = train_test_split(X_test, y_test)
    model = build_model(y_len, X_train.shape[1])
    print("done model building")
    callbacks = [
        ModelCheckpoint("model.h5",
                        save_best_only=False,
                        monitor="val_loss",
                        mode="min"),
        ModelCheckpoint("model_best.h5",
                        save_best_only=True,
                        monitor="val_loss",
                        mode="min")
    ]

    model.fit_generator(generate_data(X_train, y_train),
                        validation_data=generate_data(X_val, y_val),
                        steps_per_epoch=1000,
                        validation_steps=5,
                        epochs=10,
                        callbacks=callbacks)
    print("done training")

    print("evaluating...")
    y_pred = [np.argmax(x) for x in model.predict(X_test)]
    y_test = [np.argmax(x) for x in y_test]
    with open("eval.txt", "w") as f:
        f.write(classification_report(y_test, y_pred))
    print("done eval")

精度

                                             precision recall    f1-score  support   

AcademicConference                           0.67      0.86      0.75      14        
AcademicJournal                              0.98      0.97      0.97      1252      
Actor                                        0.49      0.49      0.49      810       
AdministrativeRegion                         0.89      0.90      0.90      3979      
AdultActor                                   0.86      0.67      0.75      161       
Aircraft                                     0.99      0.99      0.99      1932      
Airline                                      0.99      0.99      0.99      696       
Airport                                      0.98      0.96      0.97      2509      
Album                                        0.99      1.00      1.00      22773     
AmateurBoxer                                 0.36      0.72      0.48      72        
Ambassador                                   0.46      0.35      0.40      130       
AmericanFootballLeague                       0.64      0.41      0.50      17        
AmericanFootballPlayer                       0.80      0.98      0.88      3584      
AmericanFootballTeam                         0.50      1.00      0.67      5         
Amphibian                                    0.94      0.93      0.94      636       
AmusementParkAttraction                      0.74      0.91      0.82      90        
AnatomicalStructure                          0.60      0.88      0.71      309       
Animal                                       0.79      0.77      0.78      1240      
AnimangaCharacter                            0.89      0.91      0.90      34        
Anime                                        0.63      0.81      0.71      193       
Arachnid                                     0.81      0.93      0.87      745       
Archaea                                      0.80      0.86      0.83      42        
Architect                                    0.52      0.73      0.61      520       
ArchitecturalStructure                       0.47      0.56      0.51      52        
Artery                                       0.93      0.94      0.93      53        
ArtificialSatellite                          0.94      0.99      0.96      438       
Artist                                       0.53      0.52      0.53      2765      
ArtistDiscography                            0.98      0.98      0.98      721       
Artwork                                      0.92      0.94      0.93      911       
Asteroid                                     1.00      0.14      0.25      7         
Astronaut                                    0.82      0.92      0.87      115       
Athlete                                      0.91      0.88      0.90      6782      
AustralianFootballTeam                       0.59      0.93      0.72      70        
AustralianRulesFootballPlayer                0.99      1.00      0.99      2280      
AutoRacingLeague                             0.00      0.00      0.00      2         
Automobile                                   0.95      0.97      0.96      999       
AutomobileEngine                             0.90      0.96      0.93      68        
Award                                        0.89      0.96      0.92      946       
Bacteria                                     0.74      0.15      0.25      187       
BadmintonPlayer                              0.95      0.99      0.97      267       
Band                                         0.86      0.97      0.91      6034      
Bank                                         0.65      0.74      0.69      516       
Baronet                                      0.63      0.88      0.73      130       
BaseballLeague                               0.90      0.86      0.88      51        
BaseballPlayer                               0.98      0.99      0.99      4023      
BaseballSeason                               0.91      0.70      0.79      43        
BaseballTeam                                 0.38      1.00      0.55      3         
BasketballLeague                             0.74      0.89      0.81      98        
BasketballPlayer                             0.95      0.97      0.96      2193      
BasketballTeam                               0.86      0.95      0.90      278       
Bay                                          0.00      0.00      0.00      1         
BeachVolleyballPlayer                        0.83      0.13      0.22      39        
BeautyQueen                                  0.78      0.97      0.87      392       
Beverage                                     0.79      0.83      0.81      179       
BiologicalDatabase                           0.90      0.97      0.94      67        
Bird                                         0.97      0.99      0.98      2455      
BodyOfWater                                  0.66      0.54      0.60      228       
Bodybuilder                                  0.82      0.75      0.78      53        
Bone                                         0.87      0.55      0.67      62        
Book                                         0.96      0.97      0.97      6533      
Boxer                                        0.90      0.91      0.90      733       
Brain                                        0.95      0.70      0.80      99        
Brewery                                      0.25      0.09      0.13      45        
Bridge                                       0.91      0.94      0.92      751       
BroadcastNetwork                             0.65      0.24      0.35      218       
Building                                     0.76      0.80      0.78      8573      
BusCompany                                   0.75      0.90      0.82      256       
BusinessPerson                               0.38      0.05      0.08      130       
CanadianFootballTeam                         0.65      0.70      0.68      37        
Canal                                        0.80      0.94      0.87      79        
Canoeist                                     0.86      0.93      0.89      72        
Cardinal                                     0.47      0.78      0.59      137       
Castle                                       0.61      0.79      0.69      265       
Cave                                         0.66      0.89      0.76      91        
Chancellor                                   0.00      0.00      0.00      19        
Cheese                                       0.95      0.72      0.82      57        
Chef                                         0.71      0.74      0.72      118       
ChemicalCompound                             0.84      0.86      0.85      1869      
ChessPlayer                                  0.81      0.96      0.88      277       
ChristianBishop                              0.77      0.87      0.82      1655      
City                                         0.79      0.83      0.81      3829      
ClassicalMusicArtist                         0.31      0.21      0.25      52        
ClassicalMusicComposition                    0.52      0.74      0.61      122       
Cleric                                       0.76      0.51      0.61      470       
ClubMoss                                     0.00      0.00      0.00      15        
College                                      0.86      0.92      0.89      13        
CollegeCoach                                 0.91      0.91      0.91      1286      
Colour                                       0.92      0.87      0.89      38        
Comedian                                     0.52      0.34      0.41      246       
ComedyGroup                                  0.44      0.33      0.38      12        
Comic                                        0.81      0.94      0.87      374       
ComicStrip                                   0.85      0.81      0.83      62        
ComicsCharacter                              0.93      0.95      0.94      710       
ComicsCreator                                0.65      0.83      0.73      531       
Company                                      0.89      0.88      0.89      9169      
ConcentrationCamp                            0.55      0.73      0.63      15        
Congressman                                  0.47      0.63      0.54      597       
Conifer                                      0.84      0.61      0.71      128       
Constellation                                1.00      0.93      0.97      15        
Continent                                    0.00      0.00      0.00      4         
Convention                                   0.68      0.70      0.69      387       
Country                                      0.78      0.73      0.75      608       
Crater                                       0.82      0.98      0.89      121       
CricketGround                                0.82      0.97      0.89      38        
CricketTeam                                  0.76      0.98      0.86      109       
Cricketer                                    0.98      0.99      0.99      3426      
Criminal                                     0.64      0.67      0.65      424       
Crustacean                                   0.92      0.80      0.86      485       
CultivatedVariety                            0.99      0.93      0.96      307       
Curler                                       0.90      0.99      0.94      150       
CurlingLeague                                0.00      0.00      0.00      2         
Currency                                     0.90      0.91      0.91      69        
Cycad                                        1.00      0.20      0.34      44        
CyclingRace                                  0.87      0.99      0.93      147       
CyclingTeam                                  0.97      0.98      0.98      62        
Cyclist                                      0.96      0.99      0.97      2097      
Dam                                          0.94      0.91      0.93      602       
DartsPlayer                                  0.98      0.93      0.96      105       
Device                                       0.91      0.98      0.94      218       
Diocese                                      0.95      1.00      0.97      601       
Disease                                      0.86      0.95      0.90      1111      
Drug                                         0.82      0.89      0.85      1143      
Earthquake                                   0.97      0.99      0.98      137       
Economist                                    0.49      0.70      0.57      240       
EducationalInstitution                       0.74      0.65      0.70      84        
Election                                     0.68      0.68      0.68      25        
Embryology                                   0.70      0.17      0.28      40        
Engineer                                     0.44      0.53      0.48      137       
Entomologist                                 0.49      0.63      0.55      76        
Enzyme                                       0.97      0.97      0.97      939       
EthnicGroup                                  0.81      0.95      0.88      885       
Eukaryote                                    0.46      0.60      0.52      218       
EurovisionSongContestEntry                   0.83      0.96      0.89      193       
Event                                        0.89      0.55      0.68      820       
Fashion                                      0.54      0.45      0.49      71        
FashionDesigner                              0.50      0.72      0.59      125       
Fern                                         0.85      0.83      0.84      179       
FictionalCharacter                           0.84      0.88      0.86      772       
FieldHockeyLeague                            0.50      0.67      0.57      3         
FigureSkater                                 0.96      0.99      0.97      549       
Film                                         0.96      0.98      0.97      19838     
FilmFestival                                 0.86      0.97      0.91      169       
Fish                                         0.94      0.97      0.95      3295      
FloweringPlant                               0.00      0.00      0.00      23        
Food                                         0.87      0.96      0.92      850       
FootballLeagueSeason                         0.91      0.98      0.94      1674      
FootballMatch                                0.93      0.91      0.92      620       
FormulaOneRacer                              0.91      0.79      0.85      146       
FormulaOneTeam                               0.94      0.94      0.94      32        
Fungus                                       0.96      0.98      0.97      2043      
GaelicGamesPlayer                            0.98      0.99      0.99      670       
Galaxy                                       0.96      1.00      0.98      155       
Game                                         0.93      0.88      0.90      254       
Garden                                       0.76      0.92      0.83      61        
Ginkgo                                       0.00      0.00      0.00      1         
GivenName                                    0.93      0.96      0.95      665       
Glacier                                      0.95      0.92      0.94      129       
Gnetophytes                                  0.00      0.00      0.00      5         
GolfCourse                                   0.89      0.98      0.93      63        
GolfLeague                                   0.50      0.33      0.40      3         
GolfPlayer                                   0.97      1.00      0.98      609       
GolfTournament                               0.99      0.99      0.99      336       
GovernmentAgency                             0.73      0.87      0.79      979       
Governor                                     0.67      0.42      0.52      518       
GrandPrix                                    0.99      0.99      0.99      289       
Grape                                        0.94      1.00      0.97      58        
GreenAlga                                    0.57      0.70      0.63      70        
GridironFootballPlayer                       0.91      0.36      0.52      1166      
Guitarist                                    0.00      0.00      0.00      25        
Gymnast                                      0.94      0.90      0.92      410       
HandballLeague                               1.00      0.25      0.40      4         
HandballPlayer                               0.93      0.97      0.95      400       
HandballTeam                                 0.79      0.97      0.87      72        
Historian                                    0.35      0.21      0.26      115       
HistoricBuilding                             0.80      0.92      0.86      1667      
HistoricPlace                                0.71      0.64      0.67      4491      
HockeyTeam                                   0.94      0.99      0.96      454       
Holiday                                      0.88      0.82      0.85      188       
HollywoodCartoon                             0.91      0.96      0.93      293       
HorseRace                                    0.98      0.99      0.99      447       
HorseRider                                   0.76      0.90      0.82      113       
HorseTrainer                                 0.64      0.62      0.63      37        
Hospital                                     0.92      0.94      0.93      534       
Hotel                                        0.64      0.80      0.71      234       
IceHockeyLeague                              0.83      0.85      0.84      53        
IceHockeyPlayer                              0.98      1.00      0.99      2787      
InformationAppliance                         0.89      0.72      0.80      218       
InlineHockeyLeague                           0.00      0.00      0.00      1         
Insect                                       0.99      0.99      0.99      24521     
Island                                       0.82      0.92      0.86      1136      
Jockey                                       0.77      0.86      0.81      102       
Journalist                                   0.46      0.28      0.35      280       
Judge                                        0.53      0.53      0.53      548       
LacrosseLeague                               1.00      0.78      0.88      9         
LacrossePlayer                               0.94      0.97      0.95      62        
Lake                                         0.91      0.95      0.93      1779      
Language                                     0.96      0.97      0.97      1487      
LaunchPad                                    0.76      0.90      0.83      21        
LawFirm                                      0.91      0.79      0.85      66        
Legislature                                  0.82      0.92      0.87      332       
Library                                      0.82      0.76      0.79      197       
Ligament                                     0.82      0.72      0.77      32        
Lighthouse                                   0.96      0.98      0.97      341       
Locomotive                                   0.99      0.98      0.99      577       
Lymph                                        0.93      0.87      0.90      15        
Magazine                                     0.86      0.93      0.90      871       
Mammal                                       0.93      0.95      0.94      1499      
Manga                                        0.96      0.87      0.92      570       
MartialArtist                                0.92      0.92      0.92      604       
Mayor                                        0.46      0.47      0.47      307       
Medician                                     0.40      0.07      0.12      88        
MemberOfParliament                           0.68      0.79      0.73      1385      
MilitaryConflict                             0.95      0.97      0.96      2443      
MilitaryPerson                               0.80      0.90      0.85      5016      
MilitaryStructure                            0.67      0.73      0.70      803       
MilitaryUnit                                 0.95      0.96      0.96      3174      
Mineral                                      0.98      0.92      0.95      266       
MixedMartialArtsEvent                        0.95      0.99      0.97      136       
Model                                        0.74      0.60      0.66      309       
Mollusca                                     0.99      0.99      0.99      4859      
Monarch                                      0.51      0.62      0.56      409       
Monument                                     0.44      0.54      0.49      107       
Moss                                         0.89      0.18      0.30      89        
Motorcycle                                   0.98      0.98      0.98      212       
MotorcycleRacingLeague                       0.00      0.00      0.00      5         
MotorcycleRider                              0.92      0.98      0.95      211       
MotorsportSeason                             0.96      0.99      0.97      534       
Mountain                                     0.91      0.93      0.92      3025      
MountainPass                                 0.93      0.95      0.94      189       
MountainRange                                0.82      0.87      0.84      461       
Murderer                                     0.00      0.00      0.00      16        
Muscle                                       1.00      0.98      0.99      47        
Museum                                       0.79      0.87      0.83      989       
MusicFestival                                0.48      0.80      0.60      69        
MusicGenre                                   0.67      0.91      0.77      221       
Musical                                      0.91      0.97      0.94      246       
MusicalArtist                                0.70      0.88      0.78      9249      
MusicalWork                                  0.47      0.69      0.56      52        
MythologicalFigure                           0.67      0.94      0.78      151       
NCAATeamSeason                               0.99      1.00      1.00      3100      
NascarDriver                                 0.82      0.95      0.88      166       
NationalCollegiateAthleticAssociationAthlete 0.50      0.03      0.05      36        
NationalFootballLeagueEvent                  0.33      0.50      0.40      2         
NationalFootballLeagueSeason                 0.97      0.99      0.98      635       
Nerve                                        0.93      0.74      0.83      58        
NetballPlayer                                0.95      0.92      0.94      39        
Newspaper                                    0.94      0.92      0.93      1189      
Noble                                        0.59      0.57      0.58      870       
Non-ProfitOrganisation                       0.46      0.02      0.04      328       
Novel                                        0.00      0.00      0.00      6         
OfficeHolder                                 0.65      0.76      0.70      12280     
OlympicEvent                                 0.96      1.00      0.98      809       
OlympicResult                                0.85      0.99      0.92      149       
Olympics                                     0.88      0.78      0.82      9         
Organisation                                 0.65      0.78      0.71      3162      
Painter                                      0.58      0.59      0.58      469       
Park                                         0.76      0.67      0.71      699       
Person                                       0.68      0.68      0.68      33152     
Philosopher                                  0.33      0.63      0.43      361       
Photographer                                 0.36      0.29      0.32      92        
Place                                        0.90      0.65      0.75      1348      
Planet                                       0.98      0.99      0.99      663       
Plant                                        0.96      0.98      0.97      9790      
Play                                         0.83      0.93      0.88      373       
PlayboyPlaymate                              0.88      0.79      0.84      48        
Poem                                         0.59      0.60      0.59      62        
Poet                                         0.32      0.37      0.34      49        
PokerPlayer                                  0.98      0.96      0.97      130       
PoliticalParty                               0.93      0.96      0.95      1343      
Politician                                   0.64      0.49      0.56      3440      
PoloLeague                                   0.75      0.43      0.55      7         
Pope                                         0.86      0.97      0.91      63        
PowerStation                                 0.89      0.95      0.92      386       
Presenter                                    0.00      0.00      0.00      21        
President                                    0.61      0.20      0.30      417       
PrimeMinister                                0.41      0.29      0.34      249       
Prison                                       0.96      0.98      0.97      260       
ProgrammingLanguage                          0.86      0.53      0.66      180       
ProtectedArea                                0.90      0.87      0.88      1738      
Protein                                      0.81      0.84      0.83      416       
PublicTransitSystem                          0.63      0.54      0.58      337       
Publisher                                    0.81      0.67      0.73      263       
RaceHorse                                    1.00      0.99      1.00      742       
Racecourse                                   0.78      0.84      0.81      45        
RacingDriver                                 0.82      0.89      0.86      455       
RadioHost                                    0.26      0.14      0.18      74        
RadioProgram                                 0.86      0.91      0.89      234       
RadioStation                                 0.98      0.99      0.99      3641      
RailwayLine                                  0.80      0.91      0.85      645       
RailwayStation                               0.84      0.82      0.83      315       
RailwayTunnel                                0.88      0.68      0.77      41        
RecordLabel                                  0.90      0.96      0.93      576       
Religious                                    0.73      0.56      0.64      199       
ReligiousBuilding                            0.80      0.67      0.73      794       
Reptile                                      0.95      0.91      0.93      878       
ResearchProject                              0.00      0.00      0.00      1         
Restaurant                                   0.84      0.84      0.84      216       
River                                        0.98      0.99      0.99      5047      
Road                                         0.97      0.99      0.98      3725      
RoadJunction                                 0.95      0.61      0.75      31        
RoadTunnel                                   0.67      0.96      0.79      52        
Rocket                                       0.74      0.81      0.78      43        
RollerCoaster                                0.99      0.98      0.99      122       
Rower                                        0.94      0.89      0.91      53        
Royalty                                      0.70      0.84      0.76      1833      
RugbyClub                                    0.92      0.95      0.94      418       
RugbyLeague                                  0.87      0.87      0.87      98        
RugbyPlayer                                  0.98      0.99      0.99      3033      
Saint                                        0.81      0.93      0.86      710       
School                                       0.95      0.97      0.96      5821      
Scientist                                    0.58      0.60      0.59      4383      
ScreenWriter                                 0.55      0.26      0.35      138       
Sea                                          0.43      0.38      0.40      8         
Senator                                      0.33      0.45      0.38      141       
Settlement                                   0.92      0.95      0.93      43480     
Ship                                         0.99      0.99      0.99      5209      
ShoppingMall                                 0.97      0.96      0.96      428       
Single                                       0.96      0.98      0.97      8900      
SiteOfSpecialScientificInterest              0.85      0.97      0.91      191       
Skater                                       0.76      0.93      0.84      99        
SkiArea                                      0.90      0.87      0.89      107       
Skier                                        0.82      0.89      0.85      449       
SnookerChamp                                 0.00      0.00      0.00      5         
SnookerPlayer                                0.74      0.96      0.84      76        
SoapCharacter                                0.96      0.91      0.93      441       
SoccerClub                                   0.97      0.99      0.98      3770      
SoccerClubSeason                             0.97      0.99      0.98      1737      
SoccerLeague                                 0.79      0.90      0.84      318       
SoccerManager                                0.93      0.79      0.86      3417      
SoccerPlayer                                 0.96      0.99      0.98      20789     
SoccerTournament                             0.78      0.94      0.85      1232      
SoftballLeague                               0.67      0.40      0.50      5         
Software                                     0.86      0.90      0.88      1926      
SolarEclipse                                 0.97      1.00      0.98      62        
Song                                         0.77      0.62      0.69      1124      
SpaceShuttle                                 0.00      0.00      0.00      6         
SpaceStation                                 0.75      0.60      0.67      5         
Spacecraft                                   0.00      0.00      0.00      3         
Species                                      0.79      0.83      0.81      1108      
SpeedwayLeague                               0.75      0.75      0.75      4         
SpeedwayRider                                0.98      0.98      0.98      135       
SpeedwayTeam                                 0.81      0.91      0.86      23        
Sport                                        0.85      0.52      0.65      42        
SportsEvent                                  0.80      0.84      0.82      215       
SportsLeague                                 0.54      0.48      0.51      77        
SportsTeam                                   0.79      0.70      0.74      455       
SportsTeamMember                             0.97      0.61      0.75      135       
SquashPlayer                                 0.95      0.93      0.94      81        
Stadium                                      0.63      0.58      0.60      1058      
Star                                         0.99      0.98      0.99      561       
Station                                      0.98      0.98      0.98      5128      
Stream                                       0.00      0.00      0.00      9         
SumoWrestler                                 0.96      0.96      0.96      80        
SupremeCourtOfTheUnitedStatesCase            0.98      1.00      0.99      500       
Surname                                      0.49      0.69      0.57      80        
Swimmer                                      0.86      0.98      0.92      1096      
TableTennisPlayer                            0.83      0.93      0.88      88        
TelevisionEpisode                            0.98      0.98      0.98      1546      
TelevisionHost                               0.00      0.00      0.00      13        
TelevisionSeason                             0.88      0.94      0.91      640       
TelevisionShow                               0.90      0.95      0.93      6608      
TelevisionStation                            0.90      0.98      0.94      1334      
TennisLeague                                 0.00      0.00      0.00      3         
TennisPlayer                                 0.97      0.99      0.98      951       
TennisTournament                             0.95      0.99      0.97      213       
Theatre                                      0.46      0.28      0.35      127       
Town                                         0.88      0.78      0.83      7785      
TradeUnion                                   0.91      0.86      0.89      298       
Train                                        0.90      0.94      0.92      272       
Tunnel                                       0.00      0.00      0.00      24        
University                                   0.92      0.90      0.91      3383      
Valley                                       0.76      0.57      0.65      23        
Vein                                         0.97      0.88      0.93      43        
Venue                                        0.51      0.56      0.53      1085      
VideoGame                                    0.99      0.99      0.99      3492      
VideogamesLeague                             0.00      0.00      0.00      1         
Village                                      0.96      0.97      0.97      30851     
VoiceActor                                   0.36      0.41      0.38      22        
Volcano                                      0.64      0.88      0.74      131       
VolleyballCoach                              0.60      0.21      0.32      14        
VolleyballLeague                             1.00      0.69      0.82      13        
VolleyballPlayer                             0.88      0.99      0.93      676       
WaterRide                                    0.83      0.29      0.43      17        
WaterwayTunnel                               0.57      1.00      0.73      4         
Weapon                                       0.92      0.95      0.93      893       
Website                                      0.63      0.60      0.61      609       
WineRegion                                   0.89      0.85      0.87      67        
Winery                                       0.86      0.90      0.88      63        
WomensTennisAssociationTournament            0.92      0.98      0.95      94        
WorldHeritageSite                            0.45      0.54      0.49      122       
Wrestler                                     0.93      0.89      0.91      618       
WrestlingEvent                               0.96      1.00      0.98      226       
Writer                                       0.49      0.55      0.52      5418      
WrittenWork                                  0.84      0.93      0.89      272       
Year                                         0.96      0.97      0.97      281       
YearInSpaceflight                            1.00      0.92      0.96      12        
owl#Thing                                    0.68      0.51      0.58      49682     

micro avg                                    0.86      0.86      0.86      586369    
macro avg                                    0.75      0.75      0.74      586369    
weighted avg                                 0.86      0.86      0.86      586369