import pandas as pd
import numpy as np
import nltk
import re

# Map the inverse document frequencies
word_idfs = {}
for word in freq_words:
    doc_count = 0
    for data in dataset:
        if word in nltk.word_tokenize(data):
            doc_count += 1
        
        word_idfs[word] = np.log(len(dataset)/(1 + doc_count))

# Map the term frequencies
tf_map = {}
for word in freq_words:
    doc_tf = []
    for data in dataset:
        frequency = 0
        for w in nltk.word_tokenize(data):
            if word == w:
                frequency += 1
        
        tf_word = frequency/len(nltk.word_tokenize(data))
        doc_tf.append(tf_word)
    
    tf_map[word] = doc_tf

# Map tfidf
tfidf_map = []
for word in tf_map.keys():
    tfidf = []
    for value in tf_map[word]:
        score = value * word_idfs[word]
        tfidf.append(score)
        
    tfidf_map.append(tfidf)

# Display as dataframe
X = np.asarray(tfidf_map)
X = np.transpose(X)

tfidf_df = pd.DataFrame(data=np.asarray(X), columns=freq_words)
print(tfidf_df.shape)
tfidf_df.head()