Exploring Hacker News Post

from csv import reader
opened_file = open("hacker_news.csv")
read_file = reader(opened_file)
hn = list(read_file)
hn_header = hn[0] # Seperate the header from the dataset for future analysis
hn = hn[1:]

def explore_data(dataset, header = [], rows_and_column = True):
    if header:
        print(header)
    print('\n')
    
    for row in dataset[:5]:
        print(row)
        print('\n')
        
    if rows_and_column:
        total_column = len(header) if header else len(dataset[0])
        print("The number of rows: ", len(dataset))
        print("The number of columns: ", total_column)
        
explore_data(hn, hn_header)

ask_posts = []
show_posts = []
other_posts = []

for row in hn:
    title = row[1]
    title = title.lower()
    
    if title.startswith('ask hn'):
        ask_posts.append(row)
    elif title.startswith('show hn'):
        show_posts.append(row)
    else:
        other_posts.append(row)
        
print("Exploring ASK posts...")
explore_data(ask_posts)
print('\n')
print("Exploring SHOW posts...")
explore_data(show_posts)

def check_and_remove(dataset, total_column):
    removed_items = []
    for row in dataset:
        if len(row) < total_column:
            removed_items.append(row)
            dataset.remove(row)
            
    return removed_items
            
removed_ask_post_rows = check_and_remove(ask_posts, 7)
print("Total rows deleted from ask_posts: ", 
      len(removed_ask_post_rows))
removed_show_post_rows = check_and_remove(show_posts, 7)
print("Total rows deleted from show_posts: ", 
      len(removed_show_post_rows))

def check_duplicate(dataset):
    duplicate_rows = []
    already_added = []
    
    for row in dataset:
        id = row[0]
        if id not in already_added:
            already_added.append(id)
        else:
            duplicate_rows.append(row)
            
    return duplicate_rows

duplicate_ask_posts = check_duplicate(ask_posts)
duplicate_show_posts = check_duplicate(show_posts)

print("Total duplicates found in ask_post: ", len(duplicate_ask_posts))
print("Total duplicates found in show_post: ", len(duplicate_show_posts))

def find_avg_comment(dataset):
    total_comments = 0
    for post in dataset:
        num_comment = int(post[4])
        total_comments += num_comment

    avg_comments = total_comments / len(dataset)
    return avg_comments

avg_ask_comments = find_avg_comment(ask_posts)
print("The average number of comments on ask posts: {:.2f}".format(avg_ask_comments))

avg_show_comments = find_avg_comment(show_posts)
print("The average number of comments on show posts: {:.2f}".format(avg_show_comments))

avg_other_comments = find_avg_comment(other_posts)
print("The average number of comments on other posts: {:.2f}".format(avg_other_comments))

import datetime as dt
result_list = []

for post in ask_posts:
    created_at = post[6]
    num_comment = post[4]
    result_list.append([created_at, num_comment])
    
print(result_list[:5])

counts_by_hour = {}
comments_by_hour = {}

for row in result_list:
    date = row[0]
    comment_num = int(row[1])
    dt_object = dt.datetime.strptime(date, "%m/%d/%Y %H:%M")
    post_hour = dt_object.hour
    
    if post_hour not in counts_by_hour:
        counts_by_hour[post_hour] = 1
        comments_by_hour[post_hour] = comment_num
    else:
        counts_by_hour[post_hour] += 1
        comments_by_hour[post_hour] += comment_num

# print the number of posts by hour in descending order
print({k:v for k, v in sorted(counts_by_hour.items(), 
                              key = lambda elem: elem[1], reverse = True)})

# print the number of comments by hour in descending order
print({k:v for k, v in sorted(comments_by_hour.items(), 
                              key = lambda elem: elem[1], reverse = True)})

avg_by_hour = []
for key, val in comments_by_hour.items():
    avg = val / counts_by_hour[key]
    avg_by_hour.append([key, avg])
    
print(avg_by_hour[:4])

sorted_by_avg_comment = sorted(avg_by_hour, key = lambda elem: elem[1], 
                               reverse = True)

print("Top 5 hours for Ask Posts Comments:")

for elem in sorted_by_avg_comment[:5]:
    dt_object = dt.datetime.strptime(str(elem[0]), "%H")
    hour = dt_object.strftime("%H:%M")
    avg_comment = elem[1]
    print("{0}: {1:.2f} average comments per post".format(hour, avg_comment))