In [1]:

    

from ontokom.preprocessing import download_preprocessing_prerequisites, text_blob_from_file, remove_stop_words, link_noun_phrases
from glob import glob
from tqdm import tqdm

download_preprocessing_prerequisites()

files = glob("Artificial_Intelligence_Articles/*.txt")

with open("preprocessed_corpus.txt", "w", encoding="utf-8") as out_file:
    for file in tqdm(files):
        blob = text_blob_from_file(file).lower()
        blob = link_noun_phrases(blob)
        blob = remove_stop_words(blob)
        out_file.write(" ".join(blob.words) + "\n")


    

    




[nltk_data] Downloading package averaged_perceptron_tagger to
[nltk_data]     C:\Users\Tora\AppData\Roaming\nltk_data...
[nltk_data]   Package averaged_perceptron_tagger is already up-to-
[nltk_data]       date!
[nltk_data] Downloading package stopwords to
[nltk_data]     C:\Users\Tora\AppData\Roaming\nltk_data...
[nltk_data]   Unzipping corpora\stopwords.zip.
[nltk_data] Downloading package brown to
[nltk_data]     C:\Users\Tora\AppData\Roaming\nltk_data...
[nltk_data]   Package brown is already up-to-date!
[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\Tora\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!






    




100%|████████████████████████████████████████| 6379/6379 [11:35<00:00,  9.18it/s]

In [3]:

    

from ontokom.embeddings import FastTextEmbeddings

embeddings = FastTextEmbeddings("embeddings.bin")
embeddings.train("preprocessed_corpus.txt", dim=50)


    

In [4]:

    

import numpy as np

print(embeddings.embedding_for("robot"))
print(np.inner(embeddings.embedding_for("robot"), embeddings.embedding_for("ai")))
print(np.inner(embeddings.embedding_for("robot"), embeddings.embedding_for("human")))


    

    




[0.4791300594806671, 0.319511353969574, -1.7399967908859253, -0.10319866240024567, 0.557856023311615, 0.03438964858651161, -0.3273705840110779, 0.6310620307922363, 0.29031193256378174, 0.20630422234535217, -0.21921761333942413, -0.6815974712371826, -0.0005322456709109247, 0.0689973384141922, 0.5768513679504395, -0.3228307068347931, 0.9156849384307861, 0.5239145755767822, 0.13256993889808655, 0.11565254628658295, 0.2667846381664276, 0.805370032787323, 0.3811863362789154, -0.35019949078559875, 1.0029668807983398, -0.10383111983537674, 0.1430196911096573, -0.03415556252002716, -0.30773890018463135, -0.11887465417385101, 0.38866063952445984, 0.3446853756904602, 0.11107096821069717, -0.9104219079017639, -0.15470489859580994, 0.4077884554862976, 0.09930206090211868, -0.656934380531311, 0.1859755665063858, 0.06775383651256561, -0.30817359685897827, -0.4743105471134186, -0.47550705075263977, 0.11590969562530518, -0.34062162041664124, 0.4463125169277191, -0.33582383394241333, 0.07848384976387024, 0.6603894233703613, 0.41554978489875793]
4.27044361673
5.51551387186

In [ ]: