New starter code

bayes.py

+import json, re
+from collections import Counter
+from math import log, inf
+from functools import cache
+from typing import List
+
+"""After training, model_dict is a global variable which is accessible inside this module"""
+
+@cache
+def tokenize(text):
+    return [y for y in [re.sub('[^a-z0-9]', '', x) for x in text.lower().split(" ")]  if len(y)]
+
+def train(dataset):
+    model_dict = {}
+    count_of_word_by_outcome = {}
+    num_data_points = len(dataset)
+    count_of_data_points_with_outcome = Counter()
+    for point in dataset:
+        name = point['name']
+        classification = point['classification']
+        count_of_data_points_with_outcome[classification] += 1
+        if classification not in count_of_word_by_outcome:
+            count_of_word_by_outcome[classification] = Counter()
+        words = set(tokenize(point['contents']))
+        for word in words:
+            count_of_word_by_outcome[classification][word] += 1
+
+    model_dict['count_of_word_by_outcome'] = count_of_word_by_outcome
+    model_dict['num_data_points'] = num_data_points
+    model_dict['count_of_data_points_with_outcome'] = count_of_data_points_with_outcome
+
+    return model_dict
+
+"""
+TODO - Implement the following functions.
+In each of these functions, you can use the MODEL variable which is a dictionary which has three members:
+    MODEL['count_of_word_by_outcome'][outcome][word] = count_of_word_in_all_documents
+    MODEL['num_data_points'] = number_of_documents_in_the_dataset
+    MODEL['count_of_data_points_with_outcome'][outcome] = number_of_documents_in_the_data_set_which_have_the_correct_outcome_as_outcome
+"""
+@cache
+def pr_outcome(outcome : str) : # Pr(outcome)
+    return 0
+
+@cache
+def pr_word_given_outcome(word : str, outcome : str, num_words_in_document : int): # Pr(word | outcome)
+    return 0
+
+def pr_outcome_given_words(words : List[str], outcome : str): # Pr(outcome | words)
+    return 0
+
+def predict(outcomes, words):
+    best = None
+    best_likelihood = -inf
+    for outcome in outcomes:
+        pr = pr_outcome_given_words(words, outcome)
+        if  pr > best_likelihood:
+            best = outcome
+            best_likelihood = pr
+    return best

emails_validate.txt

+1.txt spam
+10.txt spam
+100.txt ham
+101.txt ham
+102.txt spam
+103.txt spam
+104.txt ham
+105.txt ham
+106.txt spam
+107.txt spam
+108.txt ham
+109.txt ham
+11.txt ham
+110.txt spam
+111.txt ham
+112.txt ham
+113.txt ham
+114.txt spam
+115.txt ham
+116.txt spam
+117.txt spam
+118.txt ham
+119.txt spam
+12.txt spam
+120.txt ham
+121.txt ham
+122.txt spam
+123.txt ham
+124.txt spam
+125.txt ham
+126.txt spam
+127.txt ham
+128.txt ham
+129.txt spam
+13.txt ham
+130.txt ham
+131.txt ham
+132.txt ham
+133.txt ham
+134.txt ham
+135.txt ham
+136.txt ham
+137.txt ham
+138.txt spam
+139.txt ham
+14.txt spam
+140.txt ham
+141.txt ham
+142.txt ham
+143.txt ham
+144.txt ham
+145.txt spam
+146.txt ham
+147.txt ham
+148.txt ham
+149.txt spam
+15.txt spam
+150.txt spam
+151.txt ham
+152.txt ham
+153.txt ham
+154.txt spam
+155.txt ham
+156.txt ham
+157.txt ham
+158.txt ham
+159.txt spam
+16.txt spam
+160.txt spam
+161.txt spam
+162.txt ham
+163.txt spam
+164.txt spam
+165.txt spam
+166.txt ham
+167.txt ham
+168.txt ham
+169.txt ham
+17.txt ham
+170.txt ham
+171.txt ham
+172.txt spam
+173.txt ham
+174.txt ham
+175.txt ham
+176.txt spam
+177.txt ham
+178.txt ham
+179.txt ham
+18.txt spam
+180.txt ham
+181.txt ham
+182.txt spam
+183.txt ham
+184.txt ham
+185.txt ham
+186.txt ham
+187.txt ham
+188.txt ham
+189.txt ham
+19.txt ham
+190.txt ham
+191.txt ham
+192.txt spam
+193.txt ham
+194.txt ham
+195.txt ham
+196.txt ham
+197.txt ham
+198.txt ham
+199.txt spam
+2.txt ham
+20.txt ham
+200.txt spam
+201.txt ham
+202.txt spam
+203.txt ham
+204.txt ham
+205.txt ham
+206.txt ham
+207.txt spam
+208.txt spam
+209.txt ham
+21.txt spam
+210.txt spam
+211.txt ham
+212.txt ham
+213.txt spam
+214.txt ham
+215.txt ham
+216.txt spam
+217.txt ham
+218.txt spam
+219.txt spam
+22.txt ham
+220.txt ham
+221.txt ham
+222.txt ham
+223.txt spam
+224.txt ham
+225.txt ham
+226.txt spam
+227.txt ham
+228.txt ham
+229.txt ham
+23.txt spam
+230.txt ham
+231.txt spam
+232.txt ham
+233.txt ham
+234.txt ham
+235.txt spam
+236.txt ham
+237.txt ham
+238.txt spam
+239.txt ham
+24.txt spam
+240.txt ham
+241.txt ham
+242.txt ham
+243.txt ham
+244.txt ham
+245.txt spam
+246.txt ham
+247.txt ham
+248.txt ham
+249.txt ham
+25.txt spam
+250.txt ham
+251.txt spam
+252.txt ham
+253.txt ham
+254.txt ham
+255.txt ham
+256.txt spam
+257.txt ham
+258.txt ham
+259.txt ham
+26.txt spam
+260.txt ham
+261.txt ham
+262.txt ham
+263.txt spam
+264.txt ham
+265.txt ham
+266.txt ham
+267.txt ham
+268.txt ham
+269.txt ham
+27.txt ham
+270.txt ham
+271.txt ham
+272.txt ham
+273.txt ham
+274.txt ham
+275.txt ham
+276.txt ham
+277.txt ham
+278.txt ham
+279.txt spam
+28.txt ham
+280.txt ham
+281.txt ham
+282.txt ham
+283.txt spam
+284.txt ham
+285.txt spam
+286.txt spam
+287.txt ham
+288.txt ham
+289.txt ham
+29.txt ham
+290.txt ham
+291.txt ham
+292.txt ham
+293.txt ham
+294.txt ham
+295.txt spam
+296.txt spam
+297.txt ham
+298.txt ham
+299.txt ham
+3.txt ham
+30.txt spam
+300.txt ham
+301.txt ham
+302.txt spam
+303.txt ham
+304.txt ham
+305.txt ham
+306.txt spam
+307.txt ham
+308.txt ham
+309.txt ham
+31.txt ham
+310.txt spam
+311.txt ham
+312.txt ham
+313.txt spam
+314.txt ham
+315.txt ham
+316.txt ham
+317.txt ham
+318.txt ham
+319.txt ham
+32.txt ham
+320.txt ham
+321.txt ham
+322.txt ham
+323.txt spam
+324.txt ham
+325.txt ham
+326.txt spam
+327.txt ham
+328.txt ham
+329.txt spam
+33.txt ham
+330.txt ham
+331.txt ham
+332.txt spam
+333.txt ham
+334.txt spam
+335.txt ham
+336.txt ham
+337.txt ham
+338.txt ham
+339.txt ham
+34.txt spam
+340.txt ham
+341.txt spam
+342.txt spam
+343.txt ham
+344.txt spam
+345.txt ham
+346.txt ham
+347.txt ham
+348.txt ham
+349.txt ham
+35.txt spam
+350.txt ham
+351.txt spam
+352.txt ham
+353.txt ham
+354.txt ham
+355.txt ham
+356.txt ham
+357.txt ham
+358.txt spam
+359.txt ham
+36.txt ham
+360.txt spam
+361.txt ham
+362.txt ham
+363.txt ham
+364.txt ham
+365.txt ham
+366.txt spam
+367.txt ham
+368.txt ham
+369.txt ham
+37.txt ham
+370.txt ham
+371.txt ham
+372.txt ham
+373.txt ham
+374.txt ham
+375.txt spam
+376.txt spam
+377.txt ham
+378.txt ham
+379.txt ham
+38.txt ham
+380.txt ham
+381.txt ham
+382.txt ham
+383.txt ham
+384.txt spam
+385.txt ham
+386.txt ham
+387.txt spam
+388.txt ham
+389.txt ham
+39.txt ham
+390.txt ham
+391.txt ham
+392.txt ham
+393.txt spam
+394.txt spam
+395.txt spam
+396.txt spam
+397.txt ham
+398.txt ham
+399.txt ham
+4.txt ham
+40.txt ham
+400.txt ham
+401.txt ham
+402.txt ham
+403.txt ham
+404.txt spam
+405.txt ham
+406.txt ham
+407.txt ham
+408.txt spam
+409.txt spam
+41.txt ham
+410.txt spam
+411.txt ham
+412.txt ham
+413.txt ham
+414.txt spam
+415.txt spam
+416.txt spam
+417.txt ham
+418.txt ham
+419.txt ham
+42.txt ham
+420.txt ham
+421.txt spam
+422.txt ham
+423.txt spam
+424.txt ham
+425.txt spam
+426.txt spam
+427.txt ham
+428.txt ham
+429.txt ham
+43.txt spam
+430.txt ham
+431.txt spam
+432.txt ham
+433.txt ham
+434.txt ham
+435.txt spam
+436.txt ham
+437.txt ham
+438.txt ham
+439.txt spam
+44.txt ham
+440.txt ham
+441.txt spam
+442.txt ham
+443.txt ham
+444.txt spam
+445.txt ham
+446.txt ham
+447.txt ham
+448.txt ham
+449.txt ham
+45.txt ham
+450.txt ham
+451.txt spam
+452.txt spam
+453.txt spam
+454.txt spam
+455.txt ham
+456.txt ham
+457.txt ham
+458.txt ham
+459.txt spam
+46.txt ham
+460.txt ham
+461.txt spam
+462.txt ham
+463.txt ham
+464.txt spam
+465.txt spam
+466.txt spam
+467.txt ham
+468.txt spam
+469.txt ham
+47.txt spam
+470.txt ham
+471.txt spam
+472.txt ham
+473.txt ham
+474.txt ham
+475.txt ham
+476.txt ham
+477.txt ham
+478.txt ham
+479.txt ham
+48.txt ham
+480.txt ham
+481.txt spam
+482.txt ham
+483.txt ham
+484.txt ham
+485.txt ham
+486.txt spam
+487.txt spam
+488.txt spam
+489.txt spam
+49.txt ham
+490.txt ham
+491.txt ham
+492.txt ham
+493.txt ham
+494.txt ham
+495.txt spam
+496.txt ham
+497.txt ham
+498.txt spam
+499.txt ham
+5.txt ham
+50.txt spam
+500.txt ham
+51.txt ham
+52.txt ham
+53.txt spam
+54.txt ham
+55.txt ham
+56.txt spam
+57.txt ham
+58.txt spam
+59.txt spam
+6.txt ham
+60.txt ham
+61.txt spam
+62.txt ham
+63.txt spam
+64.txt spam
+65.txt ham
+66.txt ham
+67.txt spam
+68.txt ham
+69.txt ham
+7.txt ham
+70.txt ham
+71.txt ham
+72.txt ham
+73.txt ham
+74.txt ham
+75.txt spam
+76.txt ham
+77.txt ham
+78.txt ham
+79.txt ham
+8.txt ham
+80.txt ham
+81.txt spam
+82.txt ham
+83.txt ham
+84.txt ham
+85.txt ham
+86.txt spam
+87.txt spam
+88.txt ham
+89.txt ham
+9.txt spam
+90.txt ham
+91.txt spam
+92.txt ham
+93.txt ham
+94.txt ham
+95.txt spam
+96.txt ham
+97.txt ham
+98.txt ham
+99.txt spam

main.py

+import json, re
+from collections import Counter
+import bayes
+
+TRAIN = 'data/train.json'
+VALIDATE = 'data/validate.json'
+
+train = json.loads(open(TRAIN).read())
+validate = json.loads(open(VALIDATE).read())
+
+def test(dataset, outcomes):
+    answers = dict([x.split(" ") for x in open(dataset + "_validate.txt").read().split("\n")[:-1]])
+
+    bayes.model_dict = bayes.train(train[dataset])
+
+    correct_by_outcome = Counter()
+    incorrect_by_outcome = Counter()
+
+    for point in validate[dataset]:
+        words = set(bayes.tokenize(point['contents']))
+        prediction = bayes.predict(outcomes, words)
+        answer = answers[point['name']]
+        if prediction == answer:
+            correct_by_outcome[answer] += 1
+        else:
+            incorrect_by_outcome[answer] += 1
+
+    print(correct_by_outcome)
+    print(incorrect_by_outcome)
+
+test('tweets', set(['positive', 'negative']))
+test('emails', set(['spam', 'ham']))
+

tweets/train/positive/11216.txt

-@andyclemmensen have you seen the game on the website?! awesomness!!! go on msn
\ No newline at end of file

tweets/train/positive/11217.txt

-"@zaneology Correction: "facetious" not "sarcastic." Apologies.  Oh, and happy Mother's Day!"
\ No newline at end of file

tweets/train/positive/11218.txt

-"@judyrey Thank you, Judy! Same back at you!"
\ No newline at end of file

tweets/train/positive/11219.txt

-says I LOVE YOU MOM  http://plurk.com/p/svm0w
\ No newline at end of file

tweets/train/positive/1122.txt

-okie gonna tweet more because i am loosing you guys or girls
\ No newline at end of file

tweets/train/positive/11220.txt

-"FUCK FUCK FUCK FUCK, @bradiewebbstack please comment me back  ILY"
\ No newline at end of file

tweets/train/positive/11222.txt

-@mari_possa Happy Happy Bday Baby Girl. Love Ya!!!! Hope you get EVERYTHING you want
\ No newline at end of file

tweets/train/positive/11223.txt

-Morning all  and its a lovely day at last
\ No newline at end of file

tweets/train/positive/11224.txt

-@katelynndivine I just texted you but I hope you know you are my favorite
\ No newline at end of file

tweets/train/positive/11225.txt

-@devyra haha one day we will have a huge roadtrip
\ No newline at end of file

tweets/train/positive/11226.txt

-@RealHughJackman Wolverine is awesome.. love it! your such a great actor..
\ No newline at end of file

tweets/train/positive/11227.txt

-@CMZart you should get your work everywhere.. Articles etc. Love it!
\ No newline at end of file

tweets/train/positive/11228.txt

-HAPPY MOTHERS DAY MAMMA
\ No newline at end of file

tweets/train/positive/11229.txt

-@talindab http://twitpic.com/4m9up - ?s this your house ?? The dog is really sweet
\ No newline at end of file