streamlit-example/utils.py at master · Meliodas-Sama/streamlit-example · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
import streamlit as st

def clean(text,lang):
    """
    clean the text by:
    1. lowering the letter case
    2. replacing [@,;/(){}[]|] symbols with space
    3. removing anything but letters and spaces
    4. replacing white spaces by a single space
    5. removing spaces left at both ends of text
    """
    import re
    if lang.lower() == 'english':
        text = text.lower()
        text = re.sub(r'[@,;/(){}[]|]', ' ', text)
        text = re.sub(r'[^a-z\s]', '', text)
        text = re.sub(r'\s+', ' ', text)
        text = text.strip()
        return text
    else:
        text = re.sub(r'[إأٱآا]','ا',text)
        text = re.sub(r'[ؤئ]','ء',text)
        text = re.sub('ة','ه',text)
        text = re.sub(r'[@,;/(){}[]|]', ' ', text)
        noise = re.compile("""   ّ    | # Tashdid
                                 َ    | # Fatha
                                 ً    | # Tanwin Fath
                                 ُ    | # Damma
                                 ٌ    | # Tanwin Damm
                                 ِ    | # Kasra
                                 ٍ    | # Tanwin Kasr
                                 ْ    | # Sukun
                                 ـ   | # Tatwil/Kashida
                             """, re.VERBOSE)
        text = re.sub(noise, '', text)
        text = re.sub(r'\s+', ' ', text)
        text = text.strip()
        return text


def getStopWordsAndStemmer(lang):
    import nltk
    from nltk import ISRIStemmer, PorterStemmer, WordNetLemmatizer
    from nltk.corpus import stopwords

    nltk.download('wordnet')
    nltk.download('stopwords')

    wnl = WordNetLemmatizer()
    if lang.lower() == 'english':
        stemmer = PorterStemmer()
        stop_words = set(stopwords.words(lang.lower()))

    elif lang.lower() == 'arabic':
        stemmer = ISRIStemmer()
        with open("Arabic_StopWords.txt", "r",encoding='utf-8') as Arabic_StopWords:
            stop_words = [line for line in Arabic_StopWords]
        stop_words = set(stop_words + stopwords.words(lang.lower()))

    return stemmer, wnl, stop_words

def stem_text(data,lang):
    """
    1. Initialize Word Net Lemmatizer
    2. Get stop words and Stemmer for the given language
    3. Clean the documents and then tokenize them using split method
       then stem the documents and remove stop words and the words that have 1 character (letter)
    """


    stemmer, wnl , stop_words = getStopWordsAndStemmer(lang)

    stemmed_cleaned = []
    for document in data:
        stemmed_cleaned.append([ wnl.lemmatize(stemmer.stem(w)) for w in clean(document,lang).split() if not w in stop_words and len(w)>1])
    if len(data) == 1:
        return stemmed_cleaned[0]
    return stemmed_cleaned


def unique_terms(documents):
    """
    Getting unique words from the documents to calculate the TF-IDF
    """
    unique_words = []
    for doc in documents:
        for w in doc:
            unique_words.append(w)
    unique_words = list(set(unique_words))
    unique_words.sort()
    return unique_words

@st.cache
def get_data(file):
    """
    Get data from the given dataFrame file/URL
    """
    import pandas as pd
    dataf = pd.read_csv(file,encoding='utf8')
    dataf.columns = ['Questions', 'Answers']
    dic = dataf.to_dict()
    ques  = list(dic['Questions'].values())
    ans = list(dic['Answers'].values())
    data = [ques[i]+' '+ans[i] for i in range(len(ans)) ]
    return ques, ans, data

def checkBoolQuery(query):
    """
    Check if the given query syntax is valid
    """
    while ('not' in query):
        i = query.index('not')
        if i+1 == len(query):
            return False
        if query[i+1] in ['not','and','or']:
            return False
        query.remove('not')

    while ('and' in query):
        i = query.index('and')
        if i+1 == len(query) or i==0 or len(query)< 3:
            return False
        if query[i+1] in ['and','or'] or query[i-1] in ['not','and','or']:
            return False
        t = query[i-1]
        query.remove('and')
        query.remove(t)

    while ('or' in query):
        i = query.index('or')
        if i+1 == len(query) or i==0 or len(query)< 3:
            return False
        if query[i+1] in ['and','or'] or query[i-1] in ['not','and','or']:
            return False
        t = query[i-1]
        query.remove('or')
        query.remove(t)
    if len(query)>1:
        return False
    return True