Preprocessing.py

'''
This document 'Preprocessing.py' preprocess the document given by casefolding, tokenization, removing the punctuations.

@author: Nisanur Genc
@author: Alex Wills
'''

from nltk import word_tokenize, sent_tokenize
import os
import re

def preprocess(document):
    '''
    Preprocesses a document and returns a list of sentences in the document.
    Sentences in this list are not tokenized and still contain punctuation.
    The first and last sentences are edited specifically for the corpus generated by CreateCorpus.py with
    the Stardew Valley Wiki, whose first and last sentences are not the same as the first and last sentences
    generated by nltk's sent_tokenize.

    @param document - the document to analyze in the form of a string.

    @return - a list containing the sentences in the document.
    '''
    # Casefold
    text = document.casefold()

    # Get sentences
    sentences = sent_tokenize(text)

    # First sentence is expected to be mixed with header, so we isolate it
    firstSentence = re.findall(r"\n.+\.", sentences[0]) 
    if len(firstSentence) > 0:
        sentences[0] = firstSentence[-1][1:]    # True first sentence is the last line that ends with a "."
    
    # Cut off last 2 sentences, which aren't part of the article
    return sentences[:-2]



def tokenize_sentence(sentence):
    '''
    Tokenizes and further processes a sentence to remove punctuation and split it into a list of words.

    @param sentence - the sentence (string) to tokenize into words
    @return - a list of casefolded words without punctuation.
    '''
    sentence = sentence.casefold()

    words = []

    # Remove punctuation
    punc = '''!()-[]{};:'"’“”\,<>./?@#$%^&*_~'''
    for character in punc:
        sentence.replace(character, "")
        words = word_tokenize(sentence)

    return words


def main():
    
    # open file for testing purposes
    rootDir = os.path.dirname(__file__)

    # with testing purposes, we opened a random file here
    with open(os.path.join(os.path.dirname(__file__), "Corpus/Abigail.txt"), mode = "r", encoding="utf-8") as file:
        document = preprocess(file.read())
        print("hi")

if __name__ == "__main__":
    main()