From 4f385c1f3cad3fcf50e448bffcec5d688b6fae5c Mon Sep 17 00:00:00 2001 From: coderooz <162082159+coderooz@users.noreply.github.com> Date: Tue, 26 Mar 2024 10:57:46 +0530 Subject: [PATCH 1/6] Direct Uploading --- AsyncHandler.py | 87 +++++ DataHandlers.py | 638 +++++++++++++++++++++++++++++++ DbHandler.py | 985 ++++++++++++++++++++++++++++++++++++++++++++++++ FileHandler.py | 251 ++++++++++++ HtmlScraper2.py | 71 ++++ Readme.md | 326 ++++++++++++++++ Requester.py | 428 +++++++++++++++++++++ htmlScraper.py | 155 ++++++++ 8 files changed, 2941 insertions(+) create mode 100644 AsyncHandler.py create mode 100644 DataHandlers.py create mode 100644 DbHandler.py create mode 100644 FileHandler.py create mode 100644 HtmlScraper2.py create mode 100644 Readme.md create mode 100644 Requester.py create mode 100644 htmlScraper.py diff --git a/AsyncHandler.py b/AsyncHandler.py new file mode 100644 index 0000000..66d2cc8 --- /dev/null +++ b/AsyncHandler.py @@ -0,0 +1,87 @@ +import asyncio, threading +import random + + +class AsyncThreadHandler: + def __init__(self): + self.tasks = [] + + def create_tasks(self, func): + pass + + + +class AsyncHandlerAio: + + def __init__(self) -> None: + self.tasks = [] + + def create_task(self, task_function, *args, **kwargs) -> None: + task = asyncio.create_task(task_function(*args, **kwargs)) + self.tasks.append(task) + + async def schedule_task(self, task_function, when, *args, **kwargs): + """ + Schedule an asynchronous task to be run at a specific time or at regular intervals. + + Parameters: + - task_function: the function to be run as an asynchronous task + - when: a float or int specifying the number of seconds in the future to run the task (for a single run) or the number of seconds between runs (for a recurring task) + - *args: positional arguments to be passed to the task function + - **kwargs: keyword arguments to be passed to the task function + + Returns: + - A Task object representing the scheduled task + """ + # Create a task using the provided task function and arguments + task = asyncio.create_task(task_function(*args, **kwargs)) + + # Schedule the task to be run at the specified time or interval + if when > 0: + # Schedule the task to run once in the future + asyncio.get_event_loop().call_later(when, task) + elif when < 0: + # Schedule the task to run repeatedly at a fixed interval + asyncio.get_event_loop().call_repeatedly(-when, task) + return task + + async def stop_tasks(self): + # Stop all the asynchronous tasks in the handler as before + for task in self.tasks: + task.cancel() + for task in self.running_tasks: + task.cancel() + await asyncio.gather(*(self.tasks + self.running_tasks), return_exceptions=True) + self.tasks = [] + self.running_tasks = [] + + async def run_task(self) -> None: + while self.tasks: + while len(self.running_tasks) < self.concurrency and self.tasks: + task = self.tasks.pop(0) + self.running_tasks.append(task) + task.add_done_callback(self.running_tasks.remove) + await asyncio.wait(self.running_tasks, return_when=asyncio.FIRST_COMPLETED) + + def get_status(self) -> dict: + """Get the status of the tasks being managed by the handler""" + return {'pending': len(self.tasks), 'running': len(self.running_tasks)} + +if __name__ == "__main__": + + async def greet(name): + print(f"Hello, {name}!") + sleep_time = random.uniform(0.5, 2.0) # Generate random sleep time between 0.5 and 2.0 seconds + await asyncio.sleep(sleep_time) + print(f"Goodbye, {name}!") + + async def main(): + # Create tasks for two concurrent greetings + task1 = asyncio.create_task(greet("Alice")) + task2 = asyncio.create_task(greet("Bob")) + + # Wait for both tasks to complete + await asyncio.gather(task1, task2) + + # Run the main coroutine + asyncio.run(main()) diff --git a/DataHandlers.py b/DataHandlers.py new file mode 100644 index 0000000..ddcf717 --- /dev/null +++ b/DataHandlers.py @@ -0,0 +1,638 @@ +from typing import Dict, List, Optional, Union +import time, json, os, pytz, re +from dateutil.relativedelta import relativedelta +from datetime import datetime, time, timedelta +from collections.abc import KeysView, ValuesView + +#### For Time Based Methods #### + + +def generate_time_intervals(start_time, duration_split:int=1, duration_format:str='d', output_format:str='human', datetime_format="%Y-%m-%d %H:%M:%S", end_time=None, excludeDate=None): + """ + Generate a list of time intervals based on the provided parameters. + + Parameters: + - start_time (str|datetime): The starting point for generating time intervals. + - duration_split (int): The duration of each time interval, specified in a numerical value. Default value is `1`. + - duration_format (str): A single-character code representing the format of the duration. + Options include 's' (seconds), 'm' (minutes), 'h' (hours), + 'd' (days), 'M' (months), and 'y' (years). + Default value is `d`. + - output_format (str): The desired output format for the time intervals. Options are 'unix' or 'human'. Default value is `human`. + - datetime_format (str, optional): A string specifying the format for human-readable time intervals. + Default is "%Y-%m-%d %H:%M:%S". + - end_time (datetime, str, optional): The end point for generating time intervals. + If not provided, the default is the current time. + - excludeDate (str, datetime, list[str], optional): The date's that are to be excluded. + + Returns: + - time_intervals (list): A list of generated time intervals based on the provided parameters. + """ + formats = {'s': 'seconds', 'm': 'minutes', 'h': 'hours', 'd': 'days', 'M': 'months', 'y': 'years'} + if isinstance(start_time, str): + start_time = datetime.strptime(start_time, datetime_format) + elif isinstance(start_time, (int, float, tuple, KeysView, ValuesView, dict, list)): + raise ValueError('The given format of data is invalid. Only excepts the datetime format or a str(i.e. 20-02-2020) whose format should match with `datetime_format` param.') + + delta = relativedelta(months=duration_split) if duration_format == 'M' else timedelta(**{formats[duration_format]: duration_split}) + + current_time, time_intervals = start_time, [] + if end_time is None: + end_time = datetime.now() + + while current_time <= end_time: + if output_format == 'unix': + time_intervals.append(current_time.timestamp()) + elif output_format == 'human': + time_intervals.append(current_time.strftime(datetime_format)) + else: + raise ValueError("Invalid output format. Choose between 'unix' or 'human'.") + current_time += delta + + if excludeDate!=None: + if isinstance(excludeDate, str): + excludeDate = [excludeDate] + elif isinstance(excludeDate, datetime): + excludeDate = [excludeDate.strftime(datetime_format)] + time_intervals = [i for i in time_intervals if i not in excludeDate] + return time_intervals + +def timestamp(given_time, format= "%Y-%m-%d %H:%M:%S", time_zone=None, normalize:str='sec'): + """ + timestamp() + ----------- + Returns the date and time in the specified format. + + Args: + + """ + + if isinstance(given_time, list): + return [timestamp(i, format, time_zone, normalize) for i in given_time] + elif isinstance(given_time, (int, float)): + given_time = convert_timestamp(given_time, normalize) + if time_zone is not None: + os.environ['TZ'] = time_zone + time.tzset() + return time.strftime(format, time.localtime(given_time)) + else: raise ValueError('The parameter passed in given_time is invalid. Please provide a valid data') + +def convert_timestamp(timestamp_ms, normalization='sec'): + """ Converts the unix timestamp to the desired format like second, minute or hour.""" + ty = type(timestamp_ms) + if ty == int or ty == float: + normalization_levels = {'millisecond': 1,'second': 1000,'minute': 1000 * 60,'hour': 1000 * 60 * 60,'day': 1000 * 60 * 60 * 24} + normalization_factor = normalization_levels.get(normalization, 1) + return timestamp_ms / normalization_factor + elif ty == list: return [convert_timestamp(i,normalization) for i in timestamp_ms] + else: raise TypeError("Argument type passed is not valid.") + +def future_timestamp(interval, unit, time_zone='UTC'): + """ + Calculate a future Unix timestamp based on the provided interval and unit. + + This function calculates a Unix timestamp representing a point in the future + by adding the specified interval of time to the current moment. + + Parameters: + - interval (int): The duration of the interval in the specified time unit. + - unit (str): A string representing the time unit of the interval. + Options include 's' (seconds), 'm' (minutes), 'h' (hours), + 'd' (days), 'mo' (months), 'y' (years). + - time_zone (str, optional): The time zone for the calculation. Default is 'UTC'. + + Returns: + - future_timestamp (float): The Unix timestamp representing the future point in time. + """ + units = {'s': 'seconds','m': 'minutes','h': 'hours','d': 'days','mo':'month', 'yrs':'year'} + tz = pytz.timezone(time_zone) + unit = units.get(unit, unit) # default to the input value if not found + future_timestamp = datetime.now(tz) + timedelta(**{unit: interval}) + return future_timestamp.timestamp() + +def split_interval(interval): + parts = re.findall(r'(\d+)([smhd])', str(interval)) + if parts: + value, unit = parts[0] + return int(value), unit + else: + raise ValueError(f'Invalid interval: {interval}') + +def to_unix_timestamp(date_time, formattype='%Y-%m-%d %H:%M:%S', time_zone='UTC'): + """Convert a human-readable date and time into the Unix timestamp in the specified time zone.""" + if isinstance(date_time, str): + dt =datetime.strptime(date_time,formattype) + tz = pytz.timezone(time_zone) + dt_localized = tz.localize(dt) + unix_timestamp = dt_localized.timestamp() + return unix_timestamp + elif isinstance(date_time, list): + return [to_unix_timestamp(i, formattype, time_zone) for i in date_time] + +def to_human_readable(unix_timestamp, formattype='%Y-%m-%d %H:%M:%S', time_zone='UTC'): + """Convert a Unix timestamp into a human-readable date and time in the specified time zone and format.""" + if isinstance(unix_timestamp, (int, float)): + dt_object = datetime.fromtimestamp(unix_timestamp) + tz = pytz.timezone(time_zone) + dt_localized = tz.localize(dt_object) + human_readable_timestamp = dt_localized.strftime(formattype) + return human_readable_timestamp + elif isinstance(unix_timestamp, list): + return [to_human_readable(i, formattype, time_zone) for i in unix_timestamp] + +def get_previous_day(date:str='', format:str='%d-%m-%Y', numberof_days:int=1): + """ + get_previous_day() + ------------------- + + This method is to get the previous date of the given date. + + Parameter: + - date (str, optional): This param takes the date who's previous date is to be fetched. Default is '', and will take the current date. + - format (str): This parameter takes the format of the date passed and will return the date inthe same format. Defaults to '%d-%m-%Y'. + - numberof_days int: takes the number of days previous to the given date's date is to be fetched. Defaault is 1. + + """ + date = datetime.now() if date == '' else datetime.strptime(date, format) + date = date - timedelta(days=numberof_days) + return date.strftime(format) + +def identify_date_format(date_str:str): + """ + identify_date_format() + ---------------------- + Identifties the date method. + """ + formats = ['%Y-%m-%d','%m-%d-%Y','%d-%m-%Y','%Y/%m/%d','%m/%d/%Y','%d/%m/%Y','%Y.%m.%d','%m.%d.%Y','%d.%m.%Y','%Y %m %d','%m %d %Y','%d %m %Y'] + + for fmt in formats: + try: + datetime.strptime(date_str, fmt) + return fmt + except ValueError: + pass + + return None + +def dateFormat(date:str, format1:str, format2:str = ''): + """ + dateFormat() + ------------ + Changes the date format. + """ + try: + date = datetime.strptime(date, format1) + except: + date = datetime.strptime(date, identify_date_format(date)) + return date if format2=='' else date.strftime(format2) + + +#### For Dict Based Methods #### + +def equalizer_dict(data:List[dict], value='')->list: + """ + Make dictionaries equal by filling in missing keys and values. + + :param data: A list of dictionaries with varying keys and values. + :type data: List[dict] + + :param value: The default value to fill in missing positions in the dictionaries. + Defaults to an empty string (''). + :type value: Any, optional + + :return: A dictionary with keys present in all input dictionaries, + and values filled or padded according to the specified default value. + :rtype: list + """ + data_len = len(data) + data2 = {k:[] for k in data[0].keys()} + for i in range(data_len): + for k, v in data[i].items(): + ky = data2.keys() + if k not in ky: + if type(v) == type(value): + data2[k] = [value] * i + elif isinstance(v, (list, dict)): + data2[k] = [[]] * i + elif isinstance(v, str): + data2[k] = [''] * i + elif isinstance(v, (int, float)): + data2[k] = [0] * i + data2[k].append(v) + + for t in ky - set(data[i].keys()): + lo = data2[t][i - 1] + if type(lo) == type(value): data2[t].append(value) + elif isinstance(lo, (list, dict)): data2[t].append([]) + elif isinstance(lo, str): data2[t].append('') + elif isinstance(lo, (int, float)): data2[t].append(0) + return dlist_dict(data2) + +def dict_dimenstion_flatener(data:dict, catg): + """ + dict_dimenstion_flatener() + -------------------------- + This method is used to reducing a multi-dimention dict into a single dimention dict. + + Parameter: + - data (dict): takes the dict that is to made into a single dimention. + + """ + pass + +def dict_lister(data:list, opt:list=None)->dict: + """ + Converts a list of dictionaries with list-type values into a dictionary with lists. + + :param data: A list of dictionaries where each dictionary contains keys with list values. + :type data: list[dict] + + :param opt: An optional parameter specifying the keys to include in the resulting dictionary. + If not provided, it defaults to using all keys present in the first dictionary. + :type opt: list, optional + + :return: A dictionary where keys are from the 'opt' parameter (or all keys if 'opt' is not provided), + and values are lists containing corresponding values from the original dictionaries. + :rtype: dict + """ + t ={} + if opt == None: + opt = data[0].keys() + for i in range(len(data)): + for k,v in data[i].items(): + if k in opt: + if (k not in t): + t[k] = [None for _ in range(i)] + [v] if i > 0 else [v] + else: t[k].append(v) + if len(opt) == 1 and len(opt)!=None: return t[opt[0]] + return t + +def dict_filter(data:dict, filter:list, catg=1): + """ + This function checks if a parameter exists in the dictionay. + :param data dict: This takes the dict that is to be checked for the data presence. + :param filter list: This takes the list of values that is to be checked, if exists in the dict. + :praram catg int: This param to to set where to check the values for. either it is in the calues section of in the keys secton. + """ + dc = {} + if catg == 1: + dc = {key: val for key, val in data.items() if key in filter} + elif catg == 0: + dc = {key: val for key, val in data.items() if val not in filter} + return dc + +def flatten_dict(d, parent_key='', sep='_', catg:int=0): + """ + This function flatens the dictionary into 1d i.e convertes the dictionay like {'a':{'b':'c'}} to {'a_b':'c'}. + :param d dict: This parameter takes the dict. + + """ + items = [] + if catg == 0: + for k, v in d.items(): + new_key = f'{parent_key}{sep}{k}' if parent_key else k + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + elif isinstance(v, list): + for i in v: + if isinstance(i, dict): + items.extend(flatten_dict(i, new_key, sep=sep).items()) + else: + items.append((new_key, i)) + else: + items.append((new_key, v)) + return dict(items) + elif catg == 1: + pass + else: raise ValueError('The value passed into the parameter `catg` is not accepted. The only value accepted are `0` & `1`.') + +def dict_reoganize(data, pattern:list): + """ + dict_reoganize() + ---------------- + + This method is to reorganze the keys of the given dict. + + Args: + data (dict|list): This patameter either takes the dict or a list of dict with similar keys. + pattern (list): This parameter takes the pattern in which the dict keys are to be arranged. + + Raises: + ValueError: The data passed in the data parameter is invalid. The paramerter only accepts either dict or a list of dict. + ValueError: _description_ + + Returns: + dict|list: This method either return the reorganized dict or the list of dict reoganized. + """ + if isinstance(data, list): + return [dict_reoganize(i, pattern) for i in data if isinstance(i, dict)] + elif isinstance(data, dict): + return {key: data[key] for key in pattern} + else: + raise ValueError('The data passed in the data parameter is invalid. The paramerter only accepts either dict or a list of dict.') + +#### For List Based Methods #### + +def dlist_dict(my_dict:dict, keys:list=[]) -> list: + """ + + Convert a dictionary into a list of dictionaries. + + This function takes a dictionary and converts it into a list of dictionaries. Each dictionary in the resulting list corresponds to a set of values for selected keys from the original dictionary. + + Parameters: + ----------- + my_dict : dict + The dictionary to be converted into a list of dictionaries. + Example: {'a': ['a_v1', 'a_v2'], 'b': ['b_v1', 'b_v2']} + + keys : list, optional + Optional parameter to select specific keys to include in the resulting list. + Defaults to an empty list ([]). + + Returns: + -------- + list + A list of dictionaries where each dictionary corresponds to a set of values for selected keys. + + Example: + -------- + >>> my_dict = {'a': ['a_v1', 'a_v2'], 'b': ['b_v1', 'b_v2']} + >>> result = dlist_dict(my_dict) + >>> print(result) + # Output: [{'a': 'a_v1', 'b': 'b_v1'}, {'a': 'a_v2', 'b': 'b_v2'}] + + >>> selected_keys = ['a'] + >>> result_selected_keys = dlist_dict(my_dict, keys=selected_keys) + >>> print(result_selected_keys) + # Output: [{'a': 'a_v1'}, {'a': 'a_v2'}] + + Notes: + ------ + - If `keys` parameter is not provided, all keys from the original dictionary will be included in the resulting list of dictionaries. + - The order of dictionaries in the resulting list is determined by the order of values for the first key in the original dictionary. + """ + + keys = my_dict.keys() if keys==[] else keys + result_list = [{k: v[i] for k, v in my_dict.items() if k in keys} for i in range(len(list(my_dict.values())[0]))] + return result_list + +def list_dlist(data, keys): + """ + This method list_dlist arranged the list list i.e.([['a','n'],['b','o']) into {key1:['a','b'], key2:['n','o']} + Note:- The inter list length and the length of strings list given must be equal. + :param list data: Takes the list that is to be arranged int the dict list format. + :param list keys: Thakes the list of strings that are to be used as the dictionary keys for the arrangement. + :return [dict] + """ + dict_key = {} + if isinstance(data[0], list) and len(keys) == len(data[0]): + for quote in data: + for i, value in enumerate(quote): + dict_key.setdefault(keys[i], []).append(value) + return dict_key + +#### For other Methods #### + +def get_unique(data:list, preserve:bool=False): + """Gets the unique data of the given list. It also has the festure fo arranging the data in a assendng ordre or sorting the given data out.""" + if preserve: + r = [] + for d in data: + if d not in r: + r.append(d) + return r + else: return sorted(list(set(data))) + +def add_index(data, index_id:int=0, index_name:str='index'): + """ + The `add_index` method is to add an id/index to the list data. + Parameter:- + - data(list): Takes the list of data which is to be indexed. The data in the list must be in a `dict` format. + - index_id (int|None): This param is note from where the indexing number should start after. Default value: 0, No will start from 1. + - index_name (str): The name of the key used to assign the index value. + """ + if len(data) <= index_id: raise ValueError('The value given in param index_id is invalid.') + for i in range(index_id+1, len(data)+1): + data[i-1][index_name] = i + return data + +def remove_empty_strings(data): + """Removes empty strings from a list or tuple.""" + return [string for string in data if string != ''] + +def decode_json(data): + """Attempts to decode a string as JSON.""" + try: + return json.loads(data) + except: + return None + +def json_parser(data, pathway): + """ + This function is designed to parse data in a JSON/dictionary structure based on a specified pathway. + + Parameters: + - data (dict): The input JSON/dictionary data. + - pathway (str): The pathway specifying the keys to navigate the data. + + Returns: + The value at the specified pathway in the input data. + + Usage Examples: + 1. Simple pathway: + ```python + data = {'name': 'Alex', 'info': {'email': 'alex@gmail.com', 'age': 25}} + pathway = 'info > email' + result = json_parser(data, pathway) + print(result) # Output: 'alex@gmail.com' + ``` + + 2. Pathway with nested keys: + ```python + data = {'person': {'name': 'John', 'details': {'age': 30, 'city': 'New York'}}} + pathway = 'person > details > city' + result = json_parser(data, pathway) + print(result) # Output: 'New York' + ``` + + 3. Pathway with list indices: + ```python + data = {'people': [{'name': 'Alice'}, {'name': 'Bob'}]} + pathway = 'people > 1 > name' + result = json_parser(data, pathway) + print(result) # Output: 'Bob' + ``` + + 4. Using a list of pathways to extract multiple values: + ```python + data = {'user': {'name': 'Alex', 'email': 'alex@gmail.com', 'age': 25}} + pathways = ['user > name', 'user > email'] + result = json_parser(data, pathways) + print(result) # Output: {'name': 'Alex', 'email': 'alex@gmail.com'} + ``` + 5. Using Special keys: + ```python + data = {'user': { + 'profile': {'name': 'John','address': {'city': 'New York', 'country': 'USA'},}, + 'preferences': {'theme': 'dark', 'notifications': True}, + } + } + + pathway = { + '__pathway__': { + 'path': 'user > profile', + 'data': ['name', 'address > country'], + },'preferences': 'user > preferences', + } + + result = json_parser(data, pathway) + print(result) # {'name': 'John','address > country': 'USA','preferences': {'theme': 'dark', 'notifications': True}} + ``` + """ + + if isinstance(pathway, list): + return {i.split('>')[0]:json_parser(data, i) for i in pathway} + elif isinstance(pathway, str) and '>' in pathway: + k = data + path = pathway.split('>') + for le in range(len(path)): + i = setNum(path[le].strip()) + if i == '`list`' and isinstance(k, list): + k = [json_parser(k, f"{str(n)} > {' > '.join(path[le + 1:])}") for n in range(len(k))] + break + elif (isinstance(k, list) and isinstance(i, int) and len(k)-1 >= i) or (isinstance(k, dict) and i in k.keys()): + k = k[i] + else: + k = None + break + return k + elif isinstance(pathway, dict): + n = {} + for k,v in pathway.items(): + if k == '__pathway__': + if isinstance(v, dict): + if isinstance(v['data'], dict): + n = {**n, **{h:json_parser(data, str(v['path'] + f' > {i}')) for h,i in v['data'].items()}} + elif isinstance(v['data'], list): + n = {**n, **{i.split('>')[-1].strip():json_parser(data, v['path'] + f' > {i}') for i in v['data']}} + elif isinstance(v['data'], str): + n[k] = json_parser(data, v['data']) + elif isinstance(v, list): + n = {**n, **{i['path'].split('>')[-1].strip():json_parser(data, {'__pathway__': i}) for i in v}} + else: + n[k] = json_parser(data, v) + return n + else: raise ValueError('The pathway given is not acceptable/invalid. Please check the pathway.') + +def round_values(data): + """Rounds the values in a list, tuple, or dictionary if they are integers or floats.""" + data_type = type(data) + if data_type == list: + return [round(i) for i in data if isinstance(i, (float, int))] + elif data_type == dict: + return {key: round_values(value) for key, value in data.items()} + elif isinstance(data, (float, int)): + return round(data) + else: + print(f'The value passed is not compatible with the method. The type of value passed is {data_type}.\nPlease try again.') + +def check_difference(*lists, time_period= None): + max_length = max([len(list) for list in lists]) + time_period = time_period if (time_period!= None) else min([len(list) for list in lists]) + lists = [list + [0] * (max_length - len(list)) for list in lists] + direction = [] + for i in range(1, time_period): + diff = sum([list[i] for list in lists]) + prev_diff = sum([list[i-1] for list in lists]) + if diff > prev_diff: + direction.append(1) + elif diff < prev_diff: + direction.append(-1) + else: + direction.append(0) + return direction[:-time_period] + +def get_differences(list1, list2): + """ This function is used to get the differences beteween the two list of numbers of a list individually at the specific positions.""" + differences = [] + for i in range(len(list1)): + diff = list1[i] - list2[i] + differences.append(diff) + return differences + +def calculate_difference_percentage(num1, num2): + """This function calculates the difference in percentage between two given numbers.""" + per = [] + for i in range(len(num1)): + difference = num1[i] - num2[i] + per.append(difference / min(num1[i], num2[i]) * 100) + return per + +def check_value_exists(value, param): + """This method/function is used to determine wether a certan value exists or not in the value/data.""" + if isinstance(param, dict): return value in param.values() + elif isinstance(param, (set, str)): return value in param + else: return value in list(param) + +def valreplace(data, target:str, replace:str, keyTy:bool=False): + """ + This method/function is used for replaceing certain or wanted values in the given subject data. + + Paramerters: + ----------- + - data (str|list|dict): This + - target (str): This parameter takes the target that has to be changed. + - replace (str): This parameter takes the value that is to be replaced by the target. + - keyTy (bool): If the data provided is a key then this value, if set to `True` then will also check look into the keys of the dict and change it accordingly. + + Return: + ------- + - str | list | dict : This method will not change the value type given and will retuen the data in same type. + """ + if isinstance(data, dict): + data = {(replace if k == target else k): v for k, v in data.items()} if keyTy else {k:valreplace(v, target, replace) for k,v in data.items()} + return data + elif isinstance(data, (list, KeysView, ValuesView)): + return [valreplace(x, target, replace, keyTy) for x in data] + elif isinstance(data, str): + if keyTy: return ' '.join([word.replace(target, replace) if word == target else word for word in data.split()]) + else: return data.replace(target, replace) + else: raise ValueError(f'This method only expects list, dict or a string. Not {type(data)}') + +def space_remover(data): + """ + This method is usd for removing any spaces in a string that is in front or behind the string. + This method can work on strings , dicts and lists of string. + """ + if isinstance(data, dict): + return {k.strip(): space_remover(v) for k,v in data.items()} + elif isinstance(data, list): + return [space_remover(i) for i in data] + elif isinstance(data, str): + return data.strip() + else: return data + +def setNum(data): + """ + This metod is used for making any possible number or float that is in a string format turn into one. + """ + if isinstance(data, dict): return {k.strip(): setNum(v) for k,v in data.items()} + elif isinstance(data, list): return [setNum(i) for i in data] + elif isinstance(data, str): + try: + return int(data) + except ValueError: + try: + return float(data) + except ValueError: + return data + else: + return data + +def get_similarities(list1, list2): + """ + This function returns the similarity between two lists and returns the simialar. + """ + return list(set(list1).intersection(set(list2))) + diff --git a/DbHandler.py b/DbHandler.py new file mode 100644 index 0000000..575bb38 --- /dev/null +++ b/DbHandler.py @@ -0,0 +1,985 @@ +import sqlite3, csv, json +import pandas as pd +from functions.DataHandlers import valreplace, equalizer_dict +from functions.FileHandler import getExtention, read, fileExists, write, write_csv +from collections.abc import KeysView, ValuesView +import mysql.connector as myqC +from datetime import datetime + +class SqliteHandler(): + """ + The `DbSqliteHandler` class simplifies interactions with SQLite3 databases in Python, offering a dynamic and efficient approach. It is designed to accelerate the development of database-related projects using Python's SQLite3 module. + + Features: + - Insert Data (insert) + - Fetch Data (fetch) + - Display Data (displayData) + - Execute Custom SQL (execute) + - Get Table Names (getTb) + - Check Index Existence (checkIndex) + - Add Index (addIndex) + - Close Database Connection + + Initialization: + --------------- + To use `DbSqliteHandler`, create an instance by providing the database name (`dbname`) and an optional path to the database directory (`dbPath`). If no path is specified, the database will be created in the current working directory. + + Example: + ```python + db_handler = SqliteHandler("my_database.db", "path/to/database/directory") + ``` + + Insert Data (insert): + --------------------- + Insert data into a specified table using the `insert` method. Provide the table name (`table`), column names (`columns`), and a list of values to be inserted (`values`). + + Example: + ```python + db_handler.insert("my_table", "column1, column2", (value1, value2)) + ``` + + Fetch Data (fetch): + ------------------- + Retrieve data from a table with the `fetch` method. Specify the table name (`table`) and optionally, specific columns to retrieve (`columns`). You can also provide a custom SQL query for advanced retrieval. + + Example: + ```python + data = db_handler.fetch("my_table", "column1, column2", "column1 = 'some_value'") + ``` + + Execute Custom SQL (execute): + ----------------------------- + Execute custom SQL queries using the `execute` method. Provide the SQL code as the `data` parameter. Use `multi=True` for executing multiple statements within a single call. + + Example: + ```python + db_handler.execute("CREATE TABLE new_table (column1 TEXT, column2 INTEGER);", multi=True) + ``` + + Database Management: + -------------------- + `DbSqliteHandler` provides various methods for managing tables and indexes, including creating, renaming, cleaning, or deleting tables. + + Example (Creating a Table): + ```python + db_handler.createTb("new_table", ["column1 TEXT", "column2 INTEGER"]) + ``` + + Closing the Connection: + ----------------------- + To close all connections and end the session, call the `close_connection` method. + + Example: + ```python + db_handler.close_connection() + ``` + + `DbSqliteHandler` offers a flexible and efficient way to interact with SQLite3 databases in Python, simplifying database-related tasks and enhancing the productivity of your projects. + """ + + def __init__(self, dbname, dbPath:str='.', json_import:bool=False, default_timeout:int=5000): + """ + Initializes the DbSqliteHandler instance. + + Parameters: + - `dbname` (str): The name of the database. + - `dbPath` (str, optional): The path to the database directory. Default is None. + """ + self.db_init = None + self.db_conn = None + self.dbName = dbname + self.dbPath = dbPath + self.dbFullPath = self.dbPath+'/'+self.dbName + ext = getExtention(self.dbFullPath) + + if json_import and ext == 'json': + self.load_dbJson(self.dbFullPath, True) + elif ext == 'db': + self.db_init = sqlite3.connect(self.dbFullPath) + self.db_conn = self.db_init.cursor() + else: + raise TypeError('File type error! only excepts json file containing dbcreating data or the db path.') + + self.DbtimeOut(default_timeout) + + def execute(self, query, data:list=[], multi: bool = False, auto_commit=True): + """ + execute() + --------- + + Executes SQLite-related code. + + Parameters: + - `query` (str): The SQL code to execute. + - `data` (list): Takes the list of values. Default is []. + - `multi` (bool): True if executing multiple statements. Default is False. + - `auto_commit` (bool): True to commit changes automatically after execution. Default is True. + + Returns: + sqlite3.Cursor: The result of the executed query. + + If Error/Or exceptions: + Prints out exceptions and rolls back the transaction if auto_commit is True. + + """ + try: + if multi==True and data!=[]: + result = self.db_conn.executemany(query, data) + else: + result = self.db_conn.execute(query) + if auto_commit: + self.db_init.commit() + return result + except sqlite3.Error as e: + print("SQLite error:", e) + if auto_commit: self.db_init.rollback() + return None + + # --- Data handleing --- # + + + def insert(self, table:str, columns, values, createTb:bool=False): + """ + Inserts data into the specified table. + + Parameters: + - `table` (str): The name of the table to insert data into. + - `columns` (str, list, KeysView): Comma-separated column names. + - `values` (list): List of values to be inserted. + - `createTb` (bool): This will create a table in the db if not available. Default is `False`. + + Returns: + sqlite3.Cursor: The result of the executed query. + """ + if createTb==True and self.getTb(table_name=table) == False: self.createTb(tbName=table, columns=columns, primary_key='id') + k=False + if isinstance(columns, (KeysView, list, tuple, ValuesView)): + keys = ['INDEX','KEY','SELECT','INSERT','UPDATE','DELETE','FROM','WHERE','JOIN','INNER','LEFT','RIGHT','GROUP BY','ORDER BY','AS','COUNT','SUM','MAX','MIN','AVG','DISTINCT','AND','OR','NOT','BETWEEN','LIKE','IN','NULL','TRUE','FALSE','TOP','LIMIT','OFFSET'] + for k in keys: + columns = valreplace(columns, k, '_'+k.upper(), 1) # type: ignore + columns = valreplace(columns, k.lower(), '_'+k.lower(), 1) # type: ignore + columns = ','.join(columns) + + if isinstance(values, (KeysView, list, tuple, ValuesView)): + value = [] + for val in values: + if isinstance(val, (KeysView, list, tuple, ValuesView)): + value.append(str(tuple(val))) + + elif isinstance(val, str): + value = str(tuple(values)) + return self.execute(f'INSERT INTO {table} ({columns}) VALUES {value};') + value = ','.join(value) + k = self.execute(f'INSERT INTO {table} ({columns}) VALUES {value} ;') + elif isinstance(values, str): + # print(values) + k = self.execute(f'INSERT INTO {table} ({columns}) VALUES ({values});') + else: raise ValueError('Invalid input format! Please provide a valid set of values.') + return k + + def json_insert(self, table_name:str, data, createTb:bool=False, ifExist:str=''): + ''' + JSON_INSERT() + ------------- + + This method is used to insert data into the table using json format data. + Parameters: + - `table_name` (str):This parametere of the method tales the name of the table in which the data is to inserted. + - `data` (list|dict): This parameter takes the data either in dict format or a list containing dicts if multiple entries are to add. + - `createTb` (bool): .. + - `ifExist` (list): A list of fields that should exist before inserting the record. If any field does not + - return None + ''' + + table_column = self.getColumnNames(table_name) + + if isinstance(data, dict): + col, query = [], [] + for k,i in data.items(): + if k in table_column and isinstance(i, (str, float, int)): + col.append(k) + query.append(i) + elif isinstance(data, list) and len(data) > 0: + data = equalizer_dict(data) + col = data[0].keys() + query = [list(i.values()) for i in data] + else:raise ValueError('The data type passed is invald. The data paramerter takes a dict or a list of dict.') + + col = list(col) + if ifExist!='': + check_data = self.fetch_unique(table_name, ifExist) + check_data = {ifExist: check_data} if isinstance(check_data, list) else check_data + for k,v in check_data.items(): + col_idx = col.index(k) + query = [val for val in query if val[col_idx] not in v] + + return self.insert(table_name, col, query, createTb) + + def update(self, table:str, updatedata, condition:str=''): + """ + update() + -------- + + This method is to update data in the table + + Parameter: + - `table`: This parameter takes the name of the table. + - `updatedata`: This parameter takes the data that is to updated. + - `"ColumnName='data1' AND ColumnName2='data2'" OR {'ColumnName':'data1', 'ColumnName2':'data2'} + ` + - `condition`: This parameter is set where the data is to be updated. + - `id='5'` + Default is ''. If left empty, then the update will happen all over the column which is specified. + + Return: + - `bool`: This method returns a boolean. True is success or False for failed opeeration.s + + Usage Example: + ``` + # assumning cl is the class like. + updatedata = "user_email='alex123@gmail.com'" + condition = "name='Alex'" + cl.update('tableName', updatedata, condition) + ``` + """ + condition = f'WHERE {condition}' if condition!='' else '' + if isinstance(updatedata, dict): + updatedata = ', '.join(["{}='{}'".format(k,v) for k,v in updatedata.items()]) + t = self.execute(f'UPDATE {table} SET {updatedata} {condition};') + if t : return True + return False + + def fetch(self, table:str, columns:str='*', query='', limit:int=0, Offset:int=0, fetchAll:bool=True,assc:str='', desc:str='', detailed:bool=True): + """ + fetch() + ------- + + Fetches data from the specified table based on the query. + + Parameters: + - `table` (str): The name of the table to fetch data from. + - `query` (str|list|dict|optional): The SQL query/Search parameter that is to be executed. + - `columns` (str): The columns that needs to be fetched. + - `limit` (int, optioanl): This parameter is to set the number of columns to fetch. + - `offset` (int, optional): The parameter if speciied will get the columns from the limit number pf columns to the number specifed in this parameter. eg: from column 5 to 23. This parameter will only be in effect of the limit parameter is use. + - `fetchAll` (bool, optional): The columns that needs to be fetched. + - `desc` (str, True, optional): The columns that are to be fetched in descending order. + + Returns: + sqlite3.Row or list of sqlite3.Row: The fetched data. + + Usage Example: + ``` + # assumning cl is the class like. + columns = "column1, column2" + query = "column1 = 'some_value'" + desc = "column2" + data = cl.fetch("my_table", columns, query, True, desc) + ``` + """ + + if self.getTb(table) == False: + raise ValueError(f'The table({table}) is not present in the database.') + + if isinstance(query, str) and query!='': + query = f' WHERE {query}' + elif isinstance(query, list)and len(query) > 0: + query = ' WHERE ' + ' AND '.join(query) + elif isinstance(query, dict)and len(query.keys()) > 0: + query = ' WHERE ' + ' AND '.join([f"{k}='{v}'" for k,v in query.items() if v!=None or v!='']) + + order = '' + if desc != '' or assc != '': + order = ' ORDER BY ' + col = self.getColumnNames(table) + if desc != '' : + order += ' DESC' if desc==True else f'{desc} DESC' if desc in col else '' + if assc != '' : + order += ' ASC' if desc==True else f'{desc} ASC' if assc in col else '' + + if limit > 0: + Offset = f' OFFSET {Offset}' if 0 < Offset < limit else '' + limit = f' LIMIT {str(limit)}' + Offset + else: + limit = '' + + ret = self.execute(f'SELECT {columns} FROM {table}{query}{order}{limit};') + try: + if ret is not None: + + if detailed==False: + if fetchAll: return ret.fetchall() + else: return ret.fetchone() + else: + col = [column[0] for column in ret.description] if ret.description else [] + if fetchAll: return [dict(zip(col, row)) for row in ret.fetchall()] + elif ret.fetchone() is not None: return dict(zip(col, ret.fetchone())) + else: return [] + except Exception as e: + print(e) + return [] + + def getTbData(self, table_name:str,columns:str='*',query:str='', limit:int=0, offset:int=0, fetchAll:bool=True, desc:str=''): + """ + Retrieves data from the specified table and returns it in a pandas DataFrame format. + + Parameters: + - `table_name` (str): The name of the table to fetch data from. + - `columns` (str, optional): The columns to retrieve. Default is '*'. + - `query` (str, optional): The SQL query or condition for data retrieval. Default is an empty string. + - `limit` (int, optional): This parameter takes the number of columns that are to be fetched. + - `offset` (int, optional): The parameter if speciied will get the columns from the limit number pf columns to the number specifed in this parameter. eg: from column 5 to 23. This parameter will only be in effect of the limit parameter is use. + - `fetchAll` (bool, optional): True to fetch all rows, False to fetch only the first row. Default is True. + - `desc` (str, optional): This parameter, if specified will get the data from the table in descending order by the name of the column mentioned. + Returns: + pd.DataFrame: A DataFrame containing the fetched data. + + Example: + ```python + data_frame = db_handler.getTbData("my_table", "column1, column2", "column1 = 'some_value'") + ``` + + Note: + This method is similar to the fetch function, but it returns the data in a pandas DataFrame format. + """ + if self.getCount(table_name) > 0: + data = self.fetch(table_name,columns, query, limit, offset, fetchAll, desc) + return pd.DataFrame(data, index=None) + else: + print(f'The table(`{table_name}`) is empty with no data.') + return False + + ### Table work/ altering related method. ### + + def fetch_unique(self, table:str, column:str): + """ + fetch_unique() + -------------- + This method fetches the unique values of a column. + + Parameter: + - table str: Takes the name of the table. + - column str: takes the name of the columns whose unique data is to fetched. + """ + column = column.split(',') + columnList = self.getColumnNames(table) + if len(column) > 1: + return {k: self.fetch_unique(table, k) for k in column if k in columnList} + elif len(column) == 1: + column = column[0] + data = self.fetch(table, f'DISTINCT {column}') + data = [i[column] for i in data] if len(data) > 0 else [] + return data + + def load_dbJson(self, data=None, fileName:bool=False)->None: + """ + This method will create the database with all its tables table and values(if provided). + """ + if fileName and fileExists(data): + self.load_dbJson(read(data, decode_json=True)) + elif isinstance(data, (str, dict)) and fileName==False: + data = json.dumps(data) if isinstance(data, str) else data + self.dbName = data['db_name'] + self.db_init = sqlite3.connect(self.dbPath+'/'+self.dbName) + self.db_conn = self.db_init.cursor() + for table in data['tables']: + print(table['table_name']) + self.createTb(table['table_name'], table['column_names']) + if table['data'] != []: + self.insert(table['table_name'], table['column_names'], table['data']) + if table['indexs'] != []: + [self.addIndex(idx['index_name'], idx['cols']) for idx in table['indexs']] + else: + raise ValueError('Check the value given passed as arguments.') + + def export_data(self, catg:str='json', tableName:str=''): + if catg=='json': + data = {'db_name': self.dbName,'tables':[],'created_on': str(datetime.now().strftime('%d-%m-%Y %H:%M:%S %p'))} + for tb in self.getTb(): + k=self.get_info(tb) + if k['rows'] > 0: + k['data'] = self.fetch(tb, detailed=False) + data['tables'].append(k) + write(f'{self.dbPath}/{self.dbName.replace('.','_')}.json', data, emptyPervious=True) + elif catg=='xls': + pass + # write(f'{self.dbPath}/{self.dbName.replace('.','_')}.csv', data, emptyPervious=True) + elif catg == 'csv': + pass + else: raise ValueError('The type of file given is not accepted.') + + def getCount(self, table_name:str, columns:str='*', query:str='')->int: + """ + getCount: + ========= + + This mehtod is to count the number of the row present in the table according the the query. + + Args: + - `table_name` (str): Takes the name of the table. + - `columns` (str) : Takes the name of the column that is to be counted. + - `query` (str) : Takes the search query by which the table is to be counted. + + Return: + int : Returns the number of rows present n tahe tble according to the query. + """ + to = f'COUNT({columns})' + col = self.fetch(table_name, to, query, detailed=False) + if col != []: + try: + return int(col[0][0]) + except: + return col + return False + + def alterTb(self, tbName:str, queryType:str, modify): + """This method is to alter tables data. + + Args: + tbName (str): parameter takes the name of the table. + queryType (str): This parameter takes the data in string format is to specify where to alter. + modify (list): This parameter takes the data in list format is to specify what to alter with. + + Returns: + _type_: _description_ + """ + if isinstance(modify, list): + modify = str(','.join(modify)) + query =f'ALTER TABLE {tbName} {queryType.upper()} {modify};' + return self.execute(query) + + def csv_insert(self, table_name:str, csv_file_path:str): + """ + Creates a table (if it doesn't exist) and adds data from a CSV file. + + Parameters: + table_name (str): The name of the table to be created or used. + csv_file_path (str): The path to the CSV file containing data to be inserted into the table. + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + if not self.getTb(table_name): + with open(csv_file_path, 'r') as csvfile: + csv_reader = csv.reader(csvfile) + headers = next(csv_reader) + column_types = ['TEXT' for _ in headers] + columns = [f"{header} {column_type}" for header, column_type in zip(headers, column_types)] + self.createTb(table_name, columns) + + with open(csv_file_path, 'r') as csvfile: + csv_reader = csv.DictReader(csvfile) + csv_data = [row for row in csv_reader] + json_data = json.dumps(csv_data, indent=2) + return self.json_insert(table_name, json_data) + except Exception as e: + print(f"Error: {e}") + return False + + def get_excel(self, tbName:str='', columns:str='*', query:str='', fetchAll:bool=True, desc:str='', fileName:str='', filePath:str='.'): + """ + This method get the specified table and saves the data in the file + + Args: + tbName (str): _description_ + columns (str, optional): _description_. Defaults to '*'. + query (str, optional): _description_. Defaults to ''. + fetchAll (bool, optional): _description_. Defaults to True. + desc (str, optional): _description_. Defaults to ''. + fileName (str, optional): _description_. Defaults to ''. + filePath (str, optional): _description_. Defaults to './'. + + Returns: + file: Returns a saved file. + """ + try: + data = self.getTbData(tbName,columns,query,fetchAll, desc) + fileName = fileName if fileName!=None else f'{tbName}.csv' + fileName = f'{filePath}/{fileName}' + data.to_csv(fileName, index=False) + return 1 + + except Exception as e: + print(e) + return 0 + + def beginTransaction(self): + """Begin a transaction.""" + self.db_init.isolation_level = None + self.execute("BEGIN TRANSACTION;") + + def commitTransaction(self): + """Commit the current transaction.""" + self.execute("COMMIT;") + self.db_init.isolation_level = '' + + def rollbackTransaction(self): + """Roll back the current transaction.""" + self.execute("ROLLBACK;") + self.db_init.isolation_level = '' # Auto-commit mode is turned on + + def getColumnNames(self, table_name): + """ + Fetches the column names of a specified table. + + Parameters: + table_name (str): The name of the table. + + Returns: + list: A list of column names. + """ + query = f"PRAGMA table_info({table_name});" + result = self.execute(query) + columns = [row[1] for row in result.fetchall()] + return columns + + def get_info(self, table_name:str=''): + """ + Retrieve information about a specific table in an SQLite database. + + Parameters: + table_name (str): The name of the table to retrieve information about. + + Returns: + dict or None: A dictionary containing table information, or None if an error occurs or the table does not exist. + + The returned dictionary contains the following keys: + - table_name: The name of the table or the entire Database. Default is ''. This means that it will give the info of the intire table + - table_description: A description of the table (if available). + - column_names: A list of column names in the table. + - column_types: A list of column data types corresponding to the column names. + """ + try: + if table_name == '': + return {i: self.get_info(i) for i in self.getTb()} + else: + columns_info = self.execute(f"PRAGMA table_info({table_name})").fetchall() + if not columns_info: return None + column_names = [info[1] for info in columns_info] + column_types = [info[2] for info in columns_info] + keyType = ['', 'PRIMARY KEY','SECONDARY KEY'] + keys = [keyType[int(info[5])] for info in columns_info] + return {'table_name': table_name,'column_names':[' '.join(k).strip() for k in zip(column_names, column_types, keys)],'rows': self.getCount(table_name), 'indexs':self.getIndexes(table_name)} + except sqlite3.Error as e: + print(f"Error: {e}") + return None + + def getTb(self, table_name:str=None): + ''' + getTb() + ------- + + This method is to get the list of all the tables present in the database. + Additional: This method also has the function to check if the table exists or not. + + Parameter: + - `table_name` (str): This parameter takes the name of the table to look for. + + Return: + - If the table_name parameter is given the either it will be returned bool true or else false. If the table_name is not added then a list of tables will be returned. + ''' + result = self.execute("SELECT name FROM sqlite_master WHERE type='table';") + data = [row[0] for row in result.fetchall()] + if table_name == None:return data + elif table_name != None and table_name in data: return True + else: return False + + def DbtimeOut(self, timeout:int=0): + if timeout == 0: + timeout = self.default_timeout + else: + self.default_timeout=timeout + + self.execute(f'PRAGMA busy_timeout = {timeout};') + + def getIndex(self, tbName:str, idxName:str): + """Check if the index exists in the table.""" + result = self.execute(f"PRAGMA index_info({idxName});") + return len(result.fetchall()) > 0 + + def getIndexes(self, tbName:str=''): + """ + getIndexes() + ------------ + + This methods gets the list of indexes. + + Parameeter: + - `tbName`: Takes the name of the table. + """ + '''This method is to get the list of the indexes related in the table.''' + if tbName: + query = f"PRAGMA index_list({tbName});" + else: + query = "PRAGMA index_list;" + result = self.execute(query) + return [row[1] for row in result.fetchall()] + + def addIndex(self, table:str, indexName:str, coloumns:str): + '''This method adds an INDEX in the table presented using the provided coloumns.''' + return self.execute(f'CREATE INDEX {indexName} ON {table} ({coloumns});') + + def delIndex(self, tbname:str, idxName:str): + '''This method is to delete an exiting index realted to a table.''' + return self.execute(f'DROP INDEX {idxName} ON {tbname};') + + def createTb(self, tbName: str, columns, primary_key: str = '', addUnique:str='', indexCol:str='', indexName:str=''): + """Creates a table in the database. + + Parameters: + tbName (str): The name of the table to be created. + columns (List[str]): A list of column names and their data types. + primary_key (str, optional): The primary key for the table. Default is 'id'. + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + keys = ['INDEX','SELECT','INSERT','UPDATE','DELETE','FROM','WHERE','JOIN','INNER','LEFT','RIGHT','GROUP BY','ORDER BY','AS','COUNT','SUM','MAX','MIN','AVG','DISTINCT','AND','OR','NOT','BETWEEN','LIKE','IN','NULL','TRUE','FALSE','TOP','LIMIT','OFFSET'] + + for k in keys: + columns = valreplace(columns, k, '_'+k.upper(), 1) + columns = valreplace(columns, k.lower(), '_'+k.lower(), 1) + + + if primary_key != '': + primary_key = primary_key if primary_key not in keys else f'_{primary_key}' + columns = [f"{primary_key} INTEGER PRIMARY KEY AUTOINCREMENT"] + columns + + if addUnique!='': + columns.append(f"UNIQUE({addUnique})") + + query = f"CREATE TABLE IF NOT EXISTS {tbName} ({', '.join(columns)});" + self.execute(query) + + if indexCol!='': + indexName = f'{tbName}_idx' if indexName=='' else indexName + self.addIndex(tbName, indexName, indexCol) + + return True + except Exception as e: + print(f"Error: {e}") + return False + + def renameTb(self, tableName1:str, tableName2:str): + """ This method is to be used to rename the tables provided in the parameter tabeName1 with the value provided in the parameter tableName2.""" + return self.alterTb(tableName1,'RENAME TO', tableName2) + + def cleanTb(self, tableName:str, query:str=''): + """This method is to be used to clean the provided tables clean of any data in it.""" + query = f'WHERE {query}' if query!='' else '' + self.execute(f'DELETE FROM {tableName}{query};') + if query == '': + self.execute(f"DELETE FROM sqlite_sequence WHERE name='{tableName}'") + else: + self.update('sqlite_sequence', {'seq': self.getCount(tableName)}, f"name='{tableName}'") + + def addUniqueColumns(self, table:str, name:str, uniques) -> None: + """ + This method will set unique value to the columns such that if user want to enter a value which already exists then it will reject. + """ + if isinstance(uniques, (list, KeysView, ValuesView)): + uniques = ','.join(uniques) + self.alterTb(table, 'ADD CONSTRAINT', f'{name} UNIQUE({uniques})') + + def delUnique(self, table_name:str, name:str)->None: + """ + This method delete the unique charectersitics of the table column constrain with the provideed name. + """ + self.alterTb(table_name, 'DROP', f'CONSTRAINT {name}') + + def delTb(self, tableName:str): + """This method is for the use of deleting tables if it exists.""" + for i in self.getIndexes(tableName): + self.delIndex(tableName, i) + return self.execute(f'DROP TABLE IF EXISTS {tableName};') + + def addColumn(self, table_name:str, new_column_name, data_type:str, adjacent_column_name:str, column_param:str='', after=True): + """ + Adds a new column to the specified table before or after a specific column. + + Parameters: + table_name (str): The name of the table to add the column to. + new_column_name (list, str, optional): The name of the new column. + data_type (str): The data type for the new column (e.g., "INTEGER", "TEXT", "REAL"). + adjacent_column_name (str): The name of the column before or after which the new column should be added. + after (bool, optional): True to add the new column after the specified column, False to add it before. Default is True. + + Returns: + sqlite3.Cursor: The result of the executed query. + """ + temp_table_name = f"temp_{table_name}" + try: + info = self.get_info(table_name) + columns = info['column_names'] + col_index = [i for i, x in enumerate(self.getColumnNames(table_name)) if adjacent_column_name == x][0] + if after: + col_index+= 1 + if isinstance(new_column_name, list): + for i in new_column_name: + columns.insert(col_index, f"{i} {data_type} {column_param}") + col_index += 1 + elif isinstance(new_column_name, str): + columns.insert(col_index, f"{new_column_name} {data_type} {column_param}") + + self.createTb(temp_table_name, columns) + if info['rows'] > 0 : + data = self.fetch(table_name) + if isinstance(new_column_name, list): + for nm in new_column_name: + data[0][nm]= '' + elif isinstance(new_column_name, str): + data[0][new_column_name]= '' + self.json_insert(temp_table_name, equalizer_dict(data)) + self.delTb(table_name) + self.renameTb(temp_table_name, table_name) + return True + except Exception as e: + self.delTb(temp_table_name) + return False + + def renameColumn(self, table_name:str, old_column_name:str, new_column_name:str): + """ + Renames a column in the specified table. + + Parameters: + table_name (str): The name of the table. + old_column_name (str): The current name of the column to be renamed. + new_column_name (str): The new name for the column. + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + columns = self.getColumnNames(table_name) + + if old_column_name not in columns: + print(f"Error: Column '{old_column_name}' not found in table '{table_name}'.") + return False + + if new_column_name in columns: + print(f"Error: Column '{new_column_name}' already exists in table '{table_name}'.") + return False + + + index = columns.index(old_column_name) + columns[index] = new_column_name + + temp_table_name = f"temp_{table_name}" + self.createTb(temp_table_name, columns) + data = valreplace(self.fetch(table_name), old_column_name, new_column_name, True) + self.json_insert(temp_table_name,) + self.delTb(table_name) + self.renameTb(temp_table_name, table_name) + + return True + except Exception as e: + print(f"Error: {e}") + return False + + def modifyColumn(self, table_name:str, column_name:str, new_column_name:str='', data_type:str='', column_param:str=''): + """ + Modifies a column in the specified table. + + Parameters: + table_name (str): The name of the table. + column_name (str): The name of the column to be modified. + new_column_name (str, optional): The new name for the column. Defaults to ''. + data_type (str, optional): The new data type for the column. Defaults to ''. + column_param (str, optional): Additional column parameters. Defaults to ''. + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + columns_info = self.execute(f"PRAGMA table_info({table_name})").fetchall() + column_names = [info[1] for info in columns_info] + + if column_name not in column_names: + print(f"Error: Column '{column_name}' not found in table '{table_name}'.") + return False + + if new_column_name == '' and data_type == '' and column_param == '': + print("Error: No modifications provided.") + return False + + temp_table_name = f"temp_{table_name}" + original_columns = self.getColumnNames(table_name) + + modified_column = f"{new_column_name} {data_type} {column_param}" if new_column_name else column_name + modified_columns = [modified_column if col == column_name else col for col in original_columns] + + self.createTb(temp_table_name, modified_columns) + self.json_insert(temp_table_name, self.fetch(table_name)) + self.delTb(table_name) + self.renameTb(temp_table_name, table_name) + + print(f"Column '{column_name}' in table '{table_name}' modified successfully.") + return True + except Exception as e: + print(f"Error: {e}") + return False + + def modifyColumns(self, table_name:str, modifications:dict): + """ + Modifies multiple columns in the specified table. + + Parameters: + table_name (str): The name of the table. + modifications (dict): A dictionary where keys are column names and values are dictionaries + containing modification options (new_column_name, data_type, column_param). + + Usage: + ``` + modifications = { + 'column1': {'new_column_name': 'new_column1', 'data_type': 'TEXT'}, + 'column2': {'data_type': 'INTEGER', 'column_param': 'NOT NULL'} + } + db_handler.modifyColumns('your_table_name', modifications) + ``` + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + for column_name, options in modifications.items(): + success = self.modifyColumn(table_name, column_name, **options) + if not success: + return False + return True + except Exception as e: + print(f"Error: {e}") + return False + + def removeColumn(self, table_name:str, column_name:str): + """ + Removes a column from the specified table. + + Parameters: + table_name (str): The name of the table to remove the column from. + column_name (str): The name of the column to remove. + + Returns: + sqlite3.Cursor: The result of the executed query. + """ + return self.execute(f"ALTER TABLE {table_name} DROP COLUMN {column_name};") + + def close_connection(self, mesg=None): + """This method for to close all the connections made to the db and aslomend the session.""" + self.db_init.close() + if mesg!=None: + print(mesg) + +class MySqlHandler(): + + def __init__(self, host:str, user:str, password:str, dataBase:str=""): + try: + self.dbConn = myqC.connect(host=host, user=user, passwd=password) + self.cursor = self.dbConn.cursor() + except myqC.Error as err: + if err.errno == myqC.errorcode.ER_ACCESS_DENIED_ERROR: + return 'Error Password!' + else: return err.errno + + + def createTb(self, tableName, columns, Engine:str='InnoDb', tableComment:str=''): + """ + """ + if self.getTables(tableName): raise ValueError(f'This table [{tableName}] already exist!') + column = ','.join(columns) + + query = f"CREATE TABLE `{tableName}` ({column}) ENGINE = \'{Engine}'" + if tableComment != '': + query += f" COMMENT = '{tableComment}'" + + query+= ';' + self.cursor.execute(query) + + def getTables(self, table:str=''): + """ + The method is to get the list of tables form the database. + """ + self.cursor.execute("SHOW TABLES") + tbList = [i[0] for i in self.cursor.fetchall()] + if table!='': return table in tbList + return tbList + + def addIndex(self, tableName:str, index_name:str, columns:list): + """ + This method is to add index to the tables. + """ + + columns = ','.join(columns) + query = f'`{index_name}` ({columns})' + self.alterTb(tableName, 'ADD UNIQUE', query) + + def alterTb(self, tableName:str, catg:str, query:str): + """ + This method is to used to alter tables in the database. + """ + self.cursor.execute(f"ALTER TABLE `{tableName}` {catg} {query};") + + def cleanTb(self, tableName:str): + """ + This method is to clean the table i.e. it will delete all the data from the table. + """ + if self.getTables(tableName): + self.cursor.execute(f'TRUNCATE TABLE `{tableName}`;') + + def delTb(self, tableName:str): + """This method is to delete the specified table.""" + if self.getTables(tableName): + self.cursor.execute(f'DROP TABLE `{tableName}`;') + + def connect_db(self, dataBase:str, create_db:bool=True): + """ + The task of this mathod is to create new databases in the server. + """ + if self.getDbList(dataBase)==False and create_db: + self.createDb(dataBase) + self.dbConn.database = dataBase + + def createDb(self, dataBase:str): + """ + This method is to crreate a new dataBase. + """ + try: + if self.getDbList(dataBase) == True: ValueError('database already exists.') + return self.execute(f'CREATE DATABASE {dataBase}') + except Exception as e: return e + + def getDbList(self, present:str=''): + """ + This method gets the list of databases present in the server. + + """ + self.execute("SHOW DATABASES") + dbList = [i[0] for i in self.cursor.fetchall()] + if present != '': return present in dbList + return dbList + + def execute(self, query:str): + """ + This method is to execute the mysql queries. + """ + try: + return self.cursor.execute(query) + except: + pass + + def delDb(self, dataBase:str): + """ + This method is to delete dataBases. + """ + if self.getDbList(dataBase): + self.execute(f'DROP DATABASE {dataBase}') + else: + raise ValueError('Database does not exist!') + + def close_connection(self): + """ + This method is to close the connection. + """ + self.cursor.close() + self.dbConn.close() \ No newline at end of file diff --git a/FileHandler.py b/FileHandler.py new file mode 100644 index 0000000..80388fd --- /dev/null +++ b/FileHandler.py @@ -0,0 +1,251 @@ +import json, csv, os +from openpyxl import Workbook, load_workbook +from PyPDF2 import PdfReader +from functions import DataHandlers as dh + + +def getFiles(file_path:str='', catg:int=0, extention=None, full_path:bool=False): + """ + + Args: + file_path (str): Takes the path of the folder that needs to be looked into. + catg (int): This parameter takes either 1 or 2, where 1 means to only look for files whereas 2 means to lik for directories. Default is 0. Which means both. + + + Returns: + (dict|list): Returns a list of files or directories in the given file. + """ + if file_path=='': + file_path = os.path.dirname(os.path.abspath(__file__)) + + files = os.listdir(file_path) + + if full_path or catg==1 or extention!=None: + files = [os.path.join(file_path, file) for file in files] + + if catg == 1 or extention != None: + if extention!=None: + files = [i for i in files if (getExtention(i)) in extention] + else: + files = [i for i in files if os.path.isfile(i)] + + elif catg == 2: + files = [i for i in files if os.path.isdir(i)] + elif catg == 0: + pass + else: raise ValueError('The parameter passed in catg is invalid. It only accepts 1 or 2 as valid parameter.') + return files if full_path else [get_only_filename(i) for i in files] + +def get_only_filename(full_path): + """_summary_ + + Args: + full_path (str): _description_ + + Returns: + str: _description_ + """ + return os.path.basename(full_path) + +def getFileCatg(file_path:str): + """ + + """ + if os.path.isdir(file_path): + return 2 + elif os.path.isfile(file_path): + return 1 + return None + +def splitFileName(file_path:str): + """_summary_ + + Args: + file_path (str): _description_ + + Returns: + tuple: _description_ + """ + return os.path.splitext(get_only_filename(file_path))[0] + +def getFileSize(file_path:str): + """_summary_ + + Args: + file_path (str): _description_ + + Returns: + bytes: _description_ + """ + + if os.path.isfile(file_path): + return os.path.getsize(file_path) + return False + +def getExtention(file:str): + """_summary_ + + Args: + file (str): _description_ + + Returns: + _type_: _description_ + """ + return os.path.splitext(file)[1].replace('.','') + +def fileExists(file:str): + """Checks if a file exists or not""" + return True if os.path.exists(file) and os.path.isfile(file) else False + +def write(file_name: str, data, separator='', emptyPervious:bool=False) -> None: + """ + Appends data to a file. + + :param file_name: The name of the file. + :param data: The data to be written to the file. + :param separator: Optional separator to append after the data. + """ + + if emptyPervious: + open(file_name, 'w').close() + + if isinstance(data,(list, tuple, set)): + [write(file_name, d, separator) for d in data] + else: + + with open(file_name, 'a', encoding='utf-8', errors='ignore') as file: + data_type = type(data) + if data_type == str: + file.write(data) + elif data_type == dict: + json.dump(data, file) + else: TypeError('The data type provided is not supported.') + file.write(separator) + +def read(file_name: str, splitter=None, encode='utf-8', error='ignore', read_bytes:bool=False, decode_json=False): + """ + Reads data from a file. + + :param file_name: The name of the file. + :param splitter: Optional splitter to split the data. + :param encode: Encoding of the file. + :param error: How to handle encoding errors. + :param decode_json: Whether to decode JSON-formatted data. + + :return: The read data. + """ + mode = 'rb' if read_bytes else 'r' + with open(file_name, mode, encoding=encode, errors=error) as file: + data = file.read() + retdata = [] + if splitter is not None: + for ret in dh.remove_empty_strings(data.split(splitter)): + if decode_json: + ret = json.loads(ret) + retdata.append(ret) + return retdata + else: + if decode_json: + return json.loads(data) + else: + return data + +def write_csv(file_name: str, data, header=None): + """ + Writes data to a CSV file. + + :param file_name: The name of the CSV file. + :param data: The data to be written to the CSV file. + :param header: Optional header for the CSV file. + """ + with open(file_name, 'w', newline='', encoding='utf-8', errors='ignore') as csvfile: + csv_writer = csv.writer(csvfile) + if header: + csv_writer.writerow(header) + csv_writer.writerows(data) + +def read_csv(file_name: str, header:bool=False): + """ + read_csv() + ---------- + Reads data from a CSV file. + + Parameter: + - file_name (str): The name of the CSV file. + - header (bool, optional): + + :return: The read data. + """ + with open(file_name, 'r', encoding='utf-8', errors='ignore') as csvfile: + csv_reader = csv.reader(csvfile) + data = [row for row in csv_reader] + if header == True: + return dh.dlist_dict(dh.list_dlist(data[1:], data[0])) + return data + +def write_excel(filename, data, sheetname='Sheet 1'): + workbook = Workbook() + sheet = workbook.active + sheet.title = sheetname + if isinstance(data, list) and isinstance(data[0], dict): + pass + elif isinstance(data, list) and isinstance(data[0], list): + for i, row in enumerate(data): + for j, value in enumerate(row): + sheet.cell(row=i+1, column=j+1, value=value) + elif isinstance(data, dict): + pass + workbook.save(filename) + +def read_excel(file_name: str, sheet_name=None, organize:bool=False): + """ + Reads data from an Excel (XLSX) file. + + :param file_name: The name of the Excel file. + :param sheet_name: Optional sheet name for the Excel file. + + :return: The read data. + """ + workbook = load_workbook(file_name) + sheet_data = {} + if sheet_name!=None and sheet_name in workbook.sheetnames: + sheet = workbook[sheet_name] + sheet_data = [list(row) for row in sheet.iter_rows(values_only=True)] + if organize: + sheet_data = dh.dlist_dict(dh.list_dlist(sheet_data[1:], sheet_data[0])) + else: + for sheet_name in workbook.sheetnames: + sheet = workbook[sheet_name] + data = [list(row) for row in sheet.iter_rows(values_only=True)] + sheet_data[sheet_name] = dh.dlist_dict(dh.list_dlist(data[1:], data[0])) if organize else data + + return sheet_data + +def read_pdf(file_name: str): + """ + Reads text data from a PDF file. + + :param file_name: The name of the PDF file. + + :return: The read text data. + """ + with open(file_name, 'rb') as pdf_file: + pdf_reader = PdfReader(pdf_file) + text = '' + for page in pdf_reader.pages: + text += page.extract_text() + return text + +def read_har(fileName:str): + entries = [] + urls = [] + data = dh.decode_json(read(fileName).replace('\n','').replace('\r', '').replace('\t', ''))['log'] + for i in data['entries'][1:]: + full_url = i['request']['url'] + if full_url not in urls and i['request']['headers'][0]['value']=='www.nseindia.com' and '/api' in full_url and full_url != "https://www.nseindia.com/api/marketStatus": + headers = {hr['name']:hr['value'] for hr in i['request']['headers']} + entries.append({'file':fileName,'full_url':full_url, 'headers':headers, 'response': i['response']['content']}) + urls.append(full_url) + return urls, entries + + diff --git a/HtmlScraper2.py b/HtmlScraper2.py new file mode 100644 index 0000000..cbeed5f --- /dev/null +++ b/HtmlScraper2.py @@ -0,0 +1,71 @@ +from bs4 import BeautifulSoup +from functions.DataHandlers import DataHandler +from functions.Requester import Requester + + +class HTMLScraper: + def __init__(self, url='', sessions:bool=True): + self.requester = Requester() + self.url = url + self.content = '' + + def _getPages(self, url:str='', method:str='get', params=None, data=None, json=None, header=None, cookies=None, timeout:int=60, redirect:bool=True, verify:bool=True, proxy=None, ref=None, agent=None, sessions=None, pre_request:bool=False): + """ + _getPages() + ----------- + + This method is for fetching webpages. + + Args: + - url (str, optional): _description_. Defaults to ''. + - method (str, optional): _description_. Defaults to 'get'. + - params (_type_, optional): _description_. Defaults to None. + - data (_type_, optional): _description_. Defaults to None. + - json (_type_, optional): _description_. Defaults to None. + - header (_type_, optional): _description_. Defaults to None. + - cookies (_type_, optional): _description_. Defaults to None. + - timeout (int, optional): _description_. Defaults to 60. + - redirect (bool, optional): _description_. Defaults to True. + - verify (bool, optional): _description_. Defaults to True. + - proxy (_type_, optional): _description_. Defaults to None. + - ref (_type_, optional): _description_. Defaults to None. + - agent (_type_, optional): _description_. Defaults to None. + - sessions (_type_, optional): _description_. Defaults to None. + - pre_request (bool, optional): _description_. Defaults to False. + + Returns: + _type_: _description_ + """ + if url == '': + url = self.url + + if sessions!=None: + response, self.sessions = self.requester.requestSessions(url, method, params, data, json, header, cookies, timeout, sessions, redirect, verify, proxy, ref, agent, pre_request) + else: + response = self.requester.request(url, method, params, data, json, header, cookies, timeout, redirect, verify, proxy, ref, agent) + return response + + def pageParser(self, content, element:str, attribute:str=''): + """ + This method is responsible for parsing the data given. + + Args: + content (_type_): _description_ + element (str): _description_ + attribute (str, optional): _description_. Defaults to ''. + + Returns: + _type_: _description_ + """ + soup = BeautifulSoup(content, 'html.parser') + elements = soup.select(element) + + if attribute: + return [element.get(attribute) for element in elements] + else: + return [element.text.strip() for element in elements] + +if __name__ == '__main__': + pass + + diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..ed2c7b0 --- /dev/null +++ b/Readme.md @@ -0,0 +1,326 @@ +# Personal EveryDay Usage Functions + +## Project Author Details: +- Name: Ranit Saha +- Code Name: Codezees +- Guthub Profile: https://github.com/Codezees + + +## Contents: + +1. [Project Info](#project_info) +2. [Project Files](#project_files) +3. [Files Description](#project_file_description) + +## Project Info +

This is just a personal project where I make classes and fuctions that will help me do certain task more effectiely rather that re-codig them again and again.

+ +## Project Files +Project Files are:- +- [AsyncHandler](#asyncHandler_file) +- [DataHandler](#dataHandler_file) +- [DbSqliteHandler](#dbsqlithandler_file) +- [Filehandler](#fileHandler_file) +- [HtmlScraper](#htmlScraper_file) +- [Requester](#requester_file) + +## Files Description +File Descriptions + +### AsyncHandler +

The file AsyncHandler encases a class called AsyncHandler(). This class is used to do task asuncronuslly.

+ +### DataHandler +

The file DataHandler encases a class called DataHandler(). This class is used to menial tasks that usually require rewriting of a lot of code. + +This class is used for handleling data related tasks +

+ +Functions of the class:- +- timestamp: Returns the date and time in the specified format. + +``` +data = timestamp(given_time, format= "%Y-%m-%d %H:%M:%S", time_zone=None, normalize:str='sec') +print(data) +#output: +``` + +### DbSqliteHandler +

The file DbSqliteHandler encases a class called DbSqliteHandler. This class is used creating and managing SQLITE3 databases. + +- Initializaton of database: +When initalizing the database, on emust provide the desired database name. If the database existes, the class will just connect it else if the the database doesnot exist, then the database will be created and connectionn will be established. + ```` + import DbSqliteHandler + dbConn = DbSqliteHandler('db_name.db') + ```` +- create(): +If the database is created newly, then the database will require a table to function properly, or else the db will just remain as file in the system. +So to create a table we need to use the class method called createTb(). + - Parameter: + - Usage: + ``` + tb_name = 'table_1' + columns = ['col1', 'col2'] + ## or you can specify the datatype like this. + columns = ['col1 TEXT', 'col2 INT'] + dbConn.createTb(tb_name, columns) + ``` + The method also comes with the feature of adding data along side creating the table raher than writing a special code for it and also comes with the feature to ```primary key``` while creating the table. + + ``` + col = ['col1', 'col2'] + data = ['data1', 'data2'] + dbConn.createTb(tbName, columns=col, insertData=data, addId=False, idKey='id') + ``` +- insert: + - Parameter: + - Usage: + ``` + + ``` +- json_insert: + - Parameter: + - Usage: + ``` + + ``` +- fetch: + - Parameter: + - Usage: + ``` + + ``` +- getCount: This method counts the number of rows in the specified table. This method also allows or count with special queries like `col1='tofu'`. + - Parameter: + - `table_name` : Takes the name of table. + - `query` : Takes the search query. Default is `''` meaning, there is no search parameter and will count all the rows in the table. + - Returns: `int` + - Usage: + ``` + query = "col1='tofu'" + num = dbconn.getCount(table_name, query) + print(num) ##prints out the number of row with col1 value as tofu. + ``` +- update: + - Parameter: + - Usage: + ``` + + ``` +- delTb: + - Parameter: + - Usage: + ``` + + ``` +- renameTb: + - Parameter: + - Usage: + ``` + + ``` +- getTb: + - Parameter: + - Usage: + ``` + + ``` +- getTbData: + - Parameter: + - Usage: + ``` + + ``` +- execute: + - Parameter: + - Usage: + ``` + + ``` +- alterTb: + - Parameter: + - Usage: + ``` + + ``` +- getColumnNames: + - Parameter: + - Usage: + ``` + + ``` +- get_table_info: + - Parameter: + - Usage: + ``` + + ``` +- modifyColumns: + - Parameter: + - Usage: + ``` + + ``` +- checkIndex: + - Parameter: + - Usage: + ``` + + ``` +- getIndexes: + - Parameter: + - Usage: + ``` + + ``` +- addIndex: + - Parameter: + - Usage: + ``` + + ``` +- delIndex: + - Parameter: + - Usage: + ``` + + ``` +- cleanTb: + - Parameter: + - Usage: + ``` + + ``` +- addColumn: + - Parameter: + - Usage: + ``` + + ``` +- renameColumn: + - Parameter: + - Usage: + ``` + + ``` +- removeColumn: Removes a column from the specified table. + - Parameter: + - Usage: + ``` + + ``` +- close_connection:This method for to close all the connections made to the db and also end the session. + + + +### Filehandler +

The file Filehandler encases a class called Filehandler. This class is used for writing & reading file related tasks. + +- Initialization: The initiallization is a simple process. To use it , just add `FileHandler.func_name()` and it will work. Examples are below. +- Class methods: The class method are as follows: + - `getFiles()`: Gets the list for files and folder in the specified folder. + - Parameter: + - `file_path`: Takes the path of the folder that needs to be looked into. + - `catg` : This parameter takes either 1 or 2, where 1 means to only look for files whereas 2 means to lik for directories. Default is 0. Which means both. + - Return: `list` + - Usage: + ``` + folder_path = './test_folder' #folder present in the current directory. + fi = getFiles(folder_path) # for both files and folder. + print(fi) #output: [file1.txt, dir1, ....] + ``` + - `get_only_filename()`: + - `getFileCatg()`: + - `splitFileName()`: + - `getExtention()`: + - `read()`: + - `write()`: This function is to write files and add contents to it. + - Parameter: + - `file_name` : The name of the file. + - `data`: The data to be written to the file. + - `separator`: Optional separator to append after the data. + - Usage: + ``` + data = 'This is Hello World File.' #file contents + fileName = 'test.txt' # the file name + FileHandler.write() + ``` + - `write_over()`: + - Parameter: + - `file_name` : The name of the file. + - `data`: The data to be written to the file. + - `separator`: Optional separator to append after the data. + - Usage: + ``` + data = 'This is Hello World File.' #file contents + fileName = 'test.txt' # the file name + FileHandler.write() + ``` + - `read_csv()`: + - Parameter: + - `file_name` : The name of the file. + - `data`: The data to be written to the file. + - `separator`: Optional separator to append after the data. + - Usage: + ``` + data = 'This is Hello World File.' #file contents + fileName = 'test.txt' # the file name + FileHandler.write() + ``` + - `write_csv()`: + - Parameter: + - `file_name` : The name of the file. + - `data`: The data to be written to the file. + - `separator`: Optional separator to append after the data. + - Usage: + ``` + data = 'This is Hello World File.' #file contents + fileName = 'test.txt' # the file name + FileHandler.write() + ``` + - `read_excel()`: + - Parameter: + - `file_name` : The name of the file. + - `data`: The data to be written to the file. + - `separator`: Optional separator to append after the data. + - Usage: + ``` + data = 'This is Hello World File.' #file contents + fileName = 'test.txt' # the file name + FileHandler.write() + ``` + - `write_excel()`: + - Parameter: + - `file_name` : The name of the file. + - `data`: The data to be written to the file. + - `separator`: Optional separator to append after the data. + - Usage: + ``` + data = 'This is Hello World File.' #file contents + fileName = 'test.txt' # the file name + FileHandler.write() + ``` + - `read_pdf()`: This function reads and returns the text values of the pdf. + - Parameter: + - `file_name` : The name of the file. + - Usage: + ``` + fileName = 'test.pdf' # the file name + FileHandler.read_pdf() + ``` + + +### HtmlScraper +

The file HtmlScraper encases a class called HtmlScraper. This class is used to scraping HTML webpages with the help of Requester class along with beautifulSoup. + +### Requester +

The file Requester encases a class called Requester. This class is used to request related tasks. + + +## + + + + + diff --git a/Requester.py b/Requester.py new file mode 100644 index 0000000..6c51fa7 --- /dev/null +++ b/Requester.py @@ -0,0 +1,428 @@ +import requests, random, time, websockets, aiohttp, asyncio +from urllib.parse import urlparse, parse_qsl, urlencode +from functions.FileHandler import read + +class Requester: + """ + Requester() + =========== + + Requester is a class for making HTTP & HTTPS requests easier speciall dureing the time of development. + + """ + + def __init__(self, agent:list=[], header:dict={}, proxy:list=[], ref:list=[], ref_file:str='', proxy_file:str='', agent_file:str='', set_agent:bool=True, set_header:bool=True, set_ref:bool=False, set_proxy:bool=False, break_pt:list=[]): + self.agent, self.ref, self.proxy, self.header = 0, '', 0, 0 + self.break_pt = break_pt + self.ws = None + + if agent_file != '': + self.agent = read(agent_file, '\n') + elif agent != []: + self.agent = agent + else: + self.agent = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'] + + if ref_file != '': + self.ref = read(ref_file,'\n') + elif ref is not None: + self.ref = ref + + if proxy_file != '': + self.proxy = read(proxy_file,'\n') + elif proxy != []: + self.proxy = proxy + else: + self.proxy= [] + + if set_header: + self.header = header + + # if self._check_connection()==False: + # raise ConnectionRefusedError('There is some issues with the internet Connection. Please the internet connection before performing any requests.') + + def get_proxy(self): + """ + This method gives a proxy url randomly. + """ + if self.proxy != []: return random.choice(self.proxy) + + def headers(self, agent:str='', ref:str='', header:dict={}, change:bool=False): + """ + headers() + -------- + The header method of the class gives out the headers necessay for the requests + Args: + agent (str, optional): _description_. Defaults to None. + ref (str, optional): _description_. Defaults to None. + header (dict, optional): _description_. Defaults to None. + change (bool, optional): This parameter takes in True or False which determines whether the headers will change or remain the same based on the data passed respectively. Defaults to False. + + Returns: + _type_: _description_ + """ + headers = {'connection': 'keep-alive','accept-Encoding': 'gzip, deflate, br','cache-Control': 'max-age=0','dnt': '1','upgrade-insecure-requests': '1','user-agent': '','accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9','sec-fetch-site': 'same-origin','sec-fetch-mode': 'navigate','sec-fetch-user': '?1', 'referer': '','accept-language': 'en-GB,en-US;q=0.9,en;q=0.8'} + + if (self.header!=0) and (header=={}) and (change==False): + header = self.header + elif (header!={}): + headers = {**headers, **{k:v for k,v in header.items()}} + if change: + self.header = headers + + if agent=='' and isinstance(self.agent, list): + agent = random.choice(self.agent) + elif agent!='': + agent = agent + + if ref=='' and self.ref!=[]: + if isinstance(self.ref, list): + ref=random.choice(self.ref) + elif isinstance(self.ref, str): + ref = self.ref + + headers['user-agent'] = str(agent) + headers['referer'] = str(ref) + + return headers + + def set_url_params(self, url, params:dict={}): + """Set the parameters in a URL. + + Parameters: + url (str): The base URL. + params (dict): A dictionary of parameters to set in the URL. + + Returns: + str: The URL with the parameters set. + """ + if params!={}: + params = {k: v for k, v in params.items() if v!=None or v!=''} + encoded_params = urlencode(params) + url = f"{url}?{encoded_params}" + return url + + def get_urlinfo(self, url): + """ + This method returns the info of the url. + """ + ino = urlparse(url) + return {'scheme':ino.scheme, 'hostname':ino.hostname, 'path':ino.path,'params':dict(parse_qsl(ino.params)), 'query':ino.query, 'fragment':ino.fragment} + + def parse_url_parameters(self, url:str): + """ + This method gets parameters from the url. + """ + return self.get_urlinfo(url)['params'] + + def request(self, url, method='get', params=None, data=None, json:dict={}, header:dict={}, cookies=None, timeout=5, redirect=True, verify=True, proxy=None, ref:str='', agent:str='', break_pt:list=[], setHeader:bool=False): + + break_pt = self.break_pt if break_pt is [] else break_pt + if break_pt != []: time.sleep(random.uniform(break_pt[0], break_pt[1])) + proxy = self.get_proxy() if proxy is None else proxy + header = None if header=={} else self.headers(agent, ref, header, setHeader) + + if method.lower() == 'get': + ret = requests.get(url, params=params, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + elif method.lower() == 'post': + ret = requests.post(url, params=params, data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + elif method.lower() == 'put': + ret = requests.put(url, params=params, data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + elif method.lower() == 'patch': + ret = requests.patch(url, params=params, data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + elif method.lower() == 'delete': + ret = requests.delete(url,params=params,data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + return ret #type:ignore + + def requestSessions(self, url:str, method:str='get', params=None, data=None, json=None, header=None, cookies=None, timeout:int=5, sessions=None, redirect=True, verify=True, proxy=None, ref:str='', agent:str='', pre_request:bool=False, break_pt:list=[]): + """ + This method (requestSessions) is do make request based on the sessions. + + Args: + url (str): Take the url that is to be requested. + method (str, optional): Takes the method in which the url is requested. Defaults to 'get'. + params (dict, optional): Takes the parameters of the url in a dict format. Defaults to None. + data (_type_, optional): _description_. Defaults to None. + json (_type_, optional): _description_. Defaults to None. + header (_type_, optional): _description_. Defaults to None. + cookies (_type_, optional): _description_. Defaults to None. + timeout (int, optional): This takes the time in seconds how much time will the systems wait for the respose . Defaults is 5 (5 sec). + sessions (_type_, optional): Takes the sessions data. Defaults to None. + redirect (bool, optional): _description_. Defaults to True. + verify (bool, optional): _description_. Defaults to True. + proxy (_type_, optional): _description_. Defaults to None. + ref (_type_, optional): Takes the reffered url or the url that will display where its requested froms. Defaults to None. + agent (_type_, optional): Takes the user-agent detials. Defaults to None. + pre_request (bool, optional): This parameter is responsible for adding sessions to the requsted url if given `True`. Defaults to False. + break_pt (_type_, optional): _description_. Defaults to None. + + Returns: + resonse: Returns the response of the requested url. + sessions: Returns the sessions created in the requseting process. + """ + break_pt = self.break_pt if break_pt == [] else break_pt + if break_pt != []: time.sleep(random.uniform(break_pt[0], break_pt[1])) + proxy = self.get_proxy() if proxy is None else proxy + header = self.headers(agent, ref) if header is None else header + + s = sessions if sessions!=None else requests.sessions.Session() + if header != None: s.headers.update(header) + if cookies!=None: s.cookies.update(cookies) + if proxy != None: s.proxies.update(proxy) + s.verify = verify + s.timeout = timeout + s.allow_redirects = redirect + + if pre_request and sessions==None: + if isinstance(pre_request, bool): + info = self.get_urlinfo(url) + prevon = info['scheme'] + '://'+ info['hostname'] + s.get(prevon) + else: s.get(pre_request) + + if method.lower() == 'get': + ret = s.get(url, params=params) + elif method.lower() == 'post': + ret = s.post(url, params=params, data=data, json=json) + elif method.lower() == 'put': + ret = s.put(url, params=params, data=data, json=json) + elif method.lower() == 'patch': + ret = s.patch(url, params=params, data=data, json=json) + elif method.lower() == 'delete': + ret = s.delete(url,params=params,data=data, json=json) + return ret, s + + def connect_websocket(self, url, on_message=None, on_error=None, on_close=None): + """Connect to a WebSocket server at the specified URL. + + Parameters: + url (str): The URL of the WebSocket server. + on_message (function): A function to be called when a message is received. + on_error (function): A function to be called when an error occurs. + on_close (function): A function to be called when the connection is closed. + + Returns: + websocket.WebSocket: The WebSocket connection. + """ + self.ws = websockets.WebSocketApp(url, on_message=on_message, on_error=on_error, on_close=on_close) + self.ws.run_forever() + return self.ws + + def send_websocket_message(self, message): + """Send a message over a WebSocket connection. + + Parameters: + ws (websocket.WebSocket): The WebSocket connection. + message (str): The message to send. + """ + self.ws.send(message) + + def close_websocket(self): + """Close a WebSocket connection. + + Parameters: + ws (websocket.WebSocket): The WebSocket connection. + """ + self.ws.close() + + def check_connection(self, url:str=''): + """ + check_connections() + ------------------- + + This method is to check if there is internet connection avaiable or not. + + Returns: + -------- + returns: This metho returns a bool (True or False). Returns `True` if conncetion is available and `False` if not available. + """ + try: + url = url if url!='' else 'https://www.google.com' + res = self.request(url) + res.raise_for_status() + return True + except requests.RequestException: + return False + +class AsyncRequester: + """ + AsyncRequester() + ================ + + AsyncRequester is a class for making HTTP & HTTPS requests easier especially during the time of development. + """ + + def __init__(self, agent=[], header={}, proxy=[], ref=[], ref_file='', proxy_file='', agent_file='', set_agent=True, set_header=True, set_ref=False, set_proxy=False, break_pt=[]): + self.agent, self.ref, self.proxy, self.header = 0, '', 0, 0 + self.break_pt = break_pt + + if agent_file != '': + self.agent = read(agent_file, '\n') + elif agent != []: + self.agent = agent + else: + self.agent = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'] + + if ref_file != '': + self.ref = read(ref_file, '\n') + elif ref is not None: + self.ref = ref + + if proxy_file != '': + self.proxy = read(proxy_file, '\n') + elif proxy != []: + self.proxy = proxy + else: + self.proxy = [] + + if set_header: + self.header = header + + async def get_proxy(self): + """ + This method gives a proxy url randomly. + """ + if self.proxy != []: + return random.choice(self.proxy) + + async def headers(self, agent='', ref='', header=None, change=False): + """ + The header method of the class gives out the headers necessary for the requests. + + Args: + agent (str, optional): User-agent string. Defaults to ''. + ref (str, optional): Referer URL. Defaults to ''. + header (dict, optional): Additional headers. Defaults to None. + change (bool, optional): Whether to change headers or not. Defaults to False. + + Returns: + dict: Request headers. + """ + headers = {'connection': 'keep-alive', 'accept-Encoding': 'gzip, deflate, br', 'cache-Control': 'max-age=0', 'dnt': '1', 'upgrade-insecure-requests': '1', 'user-agent': '', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'sec-fetch-site': 'same-origin', 'sec-fetch-mode': 'navigate', 'sec-fetch-user': '?1', 'referer': '', 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8'} + + if (self.header != {}) and (header is None) and (not change): + return self.header + elif (header is not None): + self.header = header + return header + else: + if (agent == '') and (type(self.agent) is list): + agent = random.choice(self.agent) + elif agent != '': + agent = agent + + if ref == '' and self.ref != []: + if isinstance(self.ref, list): + ref = random.choice(self.ref) + elif isinstance(self.ref, str): + ref = self.ref + + headers['user-agent'] = str(agent) + headers['referer'] = str(ref) + + return headers + + async def set_url_params(self, url, params={}): + """Set the parameters in a URL.""" + if params != {}: + params = {k: v for k, v in params.items() if v != None or v != ''} + encoded_params = urlencode(params) + url = f"{url}?{encoded_params}" + return url + + async def get_urlinfo(self, url): + """Return the info of the URL.""" + ino = urlparse(url) + return {'scheme':ino.scheme, 'hostname':ino.hostname, 'path':ino.path,'params':dict(parse_qsl(ino.params)), 'query':ino.query, 'fragment':ino.fragment} + + async def parse_url_parameters(self, url): + """Get parameters from the URL.""" + return await self.get_urlinfo(url)['params'] + + async def request(self, url, method='get', params=None, data=None, json=None, header=None, cookies=None, timeout=5, redirect=True, verify=True, proxy=None, ref='', agent='', break_pt=[]): + """Make an asynchronous HTTP request.""" + break_pt = self.break_pt if break_pt == [] else break_pt + if break_pt != []: + await asyncio.sleep(random.uniform(break_pt[0], break_pt[1])) + proxy = await self.get_proxy() if proxy is None else proxy + header = await self.headers(agent, ref) if header is None else header + + async with aiohttp.ClientSession() as session: + async with session.request(method, url, params=params, data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify_ssl=verify, proxy=proxy) as response: + return await response.text(), response + + async def requestSessions(self, url, method='get', params=None, data=None, json=None, header=None, cookies=None, timeout=5, sessions=None, redirect=True, verify=True, proxy=None, ref='', agent='', pre_request=False, break_pt=[]): + """Make an asynchronous HTTP request with sessions.""" + break_pt = self.break_pt if break_pt == [] else break_pt + if break_pt != []: + await asyncio.sleep(random.uniform(break_pt[0], break_pt[1])) + proxy = await self.get_proxy() if proxy is None else proxy + header = await self.headers(agent, ref) if header is None else header + + s = sessions if sessions is not None else aiohttp.ClientSession() + if header is not None: + s.headers.update(header) + if cookies is not None: + s.cookies.update(cookies) + s.connector.verify_ssl = verify + s.connector.timeout = aiohttp.ClientTimeout(total=timeout) + s.connector.allow_redirects = redirect + + if pre_request and sessions is None: + if isinstance(pre_request, bool): + info = await self.get_urlinfo(url) + prevon = info['scheme'] + '://' + info['hostname'] + await s.get(prevon) + else: + await s.get(pre_request) + + async with s.request(method, url, params=params, data=data, json=json) as response: + return await response.text(), response + + async def connect_websocket(self, url, on_message=None, on_error=None, on_close=None): + """Connect to a WebSocket server at the specified URL.""" + async with websockets.connect(url, on_message=on_message, on_error=on_error, on_close=on_close) as ws: + await ws.wait_closed() + return ws + + async def send_websocket_message(self, ws, message): + """Send a message over a WebSocket connection.""" + await ws.send(message) + + async def close_websocket(self, ws): + """Close a WebSocket connection.""" + await ws.close() + + async def check_connection(self, url=''): + """ + check_connections() + ------------------- + + This method is to check if there is an internet connection available or not. + + Args: + url (str, optional): URL to check the internet connection. Defaults to ''. + + Returns: + bool: True if connection is available, False otherwise. + """ + try: + url = url if url != '' else 'https://www.google.com' + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + response.raise_for_status() + return True + except aiohttp.ClientError: + return False + + + +if __name__ == '__main__': + requester = Requester() + # Make a synchronous HTTP GET request + response_text, response = requester.request('https://api.example.com/data') + # Print the response text + print(response_text) + + + diff --git a/htmlScraper.py b/htmlScraper.py new file mode 100644 index 0000000..e3eea1b --- /dev/null +++ b/htmlScraper.py @@ -0,0 +1,155 @@ +from functions.Requester import Requester +from bs4 import BeautifulSoup + +class HtmlScraper: + """ + A simple HTML scraper class using requests and BeautifulSoup. + + Attributes: + url (str): The URL of the webpage to scrape. + headers (dict): HTTP headers to be used in the requests. + page_content (str): The HTML content of the webpage. + + Methods: + _get_page_content(): Private method to fetch the HTML content of the webpage. + scrape_element(selector, attribute=None): Scrapes elements based on the provided CSS selector. + + Example: + url = 'https://example.com' + scraper = HtmlScraper(url) + + # Scraping text content + titles = scraper.scrape_element('h2.title') + for title in titles: + print(f'Title: {title}') + + # Scraping attribute (e.g., href) content + links = scraper.scrape_element('a.link', attribute='href') + for link in links: + print(f'Link: {link}') + """ + + def __init__(self, url): + """ + Initializes the HtmlScraper instance. + + Args: + url (str): The URL of the webpage to scrape. + """ + self.url, self.sessions = url, None + self.requester = Requester() + self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'} + self.page_content = self._get_page_content() + + def _get_page_content(self,url=None, method:str='get', params=None, data=None, json=None, header=None, cookies=None, timeout=30, sessions=None, redirect=True, verify=True, proxy=None, ref=None, agent=None, pre_request=None): + """ + Private method to fetch the HTML content of the webpage. + + Returns: + str: The HTML content of the webpage. + + Raises: + Exception: If the request to fetch the page content fails. + """ + if url==None: + url=self.url + + if header == None: + header = self.headers + + if sessions!=None: + response, self.sessions = self.requester.requestSessions(url, method, params, data, json, header, cookies, timeout, sessions, redirect, verify, proxy, ref, agent, pre_request) + else: + response = self.requester.request(url, method, params, data, json, header, cookies, timeout, redirect, verify, proxy, ref, agent) + + if response.status_code == 200: return response.text + else: raise Exception(f"Failed to fetch page content. Status code: {response.status_code}") + + def scrape_element(self, selector, attribute=None): + """ + Scrapes elements based on the provided CSS selector. + + Args: + selector (str): The CSS selector to locate the desired elements. + attribute (str, optional): The attribute to extract from the elements (e.g., 'href' for links). Defaults to None. + + Returns: + list: A list of scraped content (either text or attribute values). + """ + soup = BeautifulSoup(self.page_content, 'html.parser') + elements = soup.select(selector) + + if attribute: + return [element.get(attribute) for element in elements] + else: + return [element.text.strip() for element in elements] + + def scrape_elements_grouped(self, info_dict): + """ + Scrapes elements based on the provided URL and element pathways, and groups the data. + + Args: + info_dict (dict): A dictionary containing the URL and element pathways. + + Returns: + dict: A dictionary with specified keys and values grouped in lists. + """ + if isinstance(info_dict, dict): + url = info_dict.get('url', '') + paths = info_dict.get('path', {}) + + if url: + self._get_page_content(url) + + if not paths: + raise ValueError("URL and element pathways must be specified in the info_dict.") + + grouped_data = {} + for key, path in paths.items(): + if isinstance(path, str): + grouped_data[key] = self.scrape_element(path) + elif isinstance(path, tuple): + grouped_data[key] = self.scrape_element(path[0], path[1]) + elif isinstance(path, dict): + grouped_data[key] = self.scrape_elements_grouped(key) + elif isinstance(path, list): + for i in path: + if isinstance(i, str): + grouped_data[key] = self.scrape_element(i) + elif isinstance(i, dict): + grouped_data[key] = self.scrape_elements_grouped(i) + else: raise ValueError('The path is either needs to be string, dict or a listof those.') + + return grouped_data + elif isinstance(info_dict, list): + return [self.scrape_elements_grouped(i) for i in info_dict] + else: return False + + +if __name__ == '__main__': + # url = 'https://example.com' + # scraper = HtmlScraper(url) + # titles = scraper.scrape_element('body > div:nth-child(1) > p') + # for title in titles: + # print(f'Title: {title}') + # links = scraper.scrape_element('a', attribute='href') + # for link in links: + # print(f'Link: {link}') + + + scrape_data = { + 'url':'https://example.com', + 'params': {'continue':False}, + 'path': { + 'title':{ + 'element':'body > div:nth-child(1) > p', + 'attr': None, + 'path': {} + }, + 'link' : { + 'element':'a', + 'attr': 'href', + 'path':{} + } + } + } \ No newline at end of file From 9a3eda4911de2da1a8783da95235f371e6c9aa65 Mon Sep 17 00:00:00 2001 From: Ranit Saha <162082159+coderooz@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:59:18 +0530 Subject: [PATCH 2/6] Delete HtmlScraper2.py --- HtmlScraper2.py | 71 ------------------------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 HtmlScraper2.py diff --git a/HtmlScraper2.py b/HtmlScraper2.py deleted file mode 100644 index cbeed5f..0000000 --- a/HtmlScraper2.py +++ /dev/null @@ -1,71 +0,0 @@ -from bs4 import BeautifulSoup -from functions.DataHandlers import DataHandler -from functions.Requester import Requester - - -class HTMLScraper: - def __init__(self, url='', sessions:bool=True): - self.requester = Requester() - self.url = url - self.content = '' - - def _getPages(self, url:str='', method:str='get', params=None, data=None, json=None, header=None, cookies=None, timeout:int=60, redirect:bool=True, verify:bool=True, proxy=None, ref=None, agent=None, sessions=None, pre_request:bool=False): - """ - _getPages() - ----------- - - This method is for fetching webpages. - - Args: - - url (str, optional): _description_. Defaults to ''. - - method (str, optional): _description_. Defaults to 'get'. - - params (_type_, optional): _description_. Defaults to None. - - data (_type_, optional): _description_. Defaults to None. - - json (_type_, optional): _description_. Defaults to None. - - header (_type_, optional): _description_. Defaults to None. - - cookies (_type_, optional): _description_. Defaults to None. - - timeout (int, optional): _description_. Defaults to 60. - - redirect (bool, optional): _description_. Defaults to True. - - verify (bool, optional): _description_. Defaults to True. - - proxy (_type_, optional): _description_. Defaults to None. - - ref (_type_, optional): _description_. Defaults to None. - - agent (_type_, optional): _description_. Defaults to None. - - sessions (_type_, optional): _description_. Defaults to None. - - pre_request (bool, optional): _description_. Defaults to False. - - Returns: - _type_: _description_ - """ - if url == '': - url = self.url - - if sessions!=None: - response, self.sessions = self.requester.requestSessions(url, method, params, data, json, header, cookies, timeout, sessions, redirect, verify, proxy, ref, agent, pre_request) - else: - response = self.requester.request(url, method, params, data, json, header, cookies, timeout, redirect, verify, proxy, ref, agent) - return response - - def pageParser(self, content, element:str, attribute:str=''): - """ - This method is responsible for parsing the data given. - - Args: - content (_type_): _description_ - element (str): _description_ - attribute (str, optional): _description_. Defaults to ''. - - Returns: - _type_: _description_ - """ - soup = BeautifulSoup(content, 'html.parser') - elements = soup.select(element) - - if attribute: - return [element.get(attribute) for element in elements] - else: - return [element.text.strip() for element in elements] - -if __name__ == '__main__': - pass - - From fdaa9a19ba4eef222a55b6cb22a3af933d2211d5 Mon Sep 17 00:00:00 2001 From: Ranit Saha <162082159+coderooz@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:59:40 +0530 Subject: [PATCH 3/6] Delete htmlScraper.py --- htmlScraper.py | 155 ------------------------------------------------- 1 file changed, 155 deletions(-) delete mode 100644 htmlScraper.py diff --git a/htmlScraper.py b/htmlScraper.py deleted file mode 100644 index e3eea1b..0000000 --- a/htmlScraper.py +++ /dev/null @@ -1,155 +0,0 @@ -from functions.Requester import Requester -from bs4 import BeautifulSoup - -class HtmlScraper: - """ - A simple HTML scraper class using requests and BeautifulSoup. - - Attributes: - url (str): The URL of the webpage to scrape. - headers (dict): HTTP headers to be used in the requests. - page_content (str): The HTML content of the webpage. - - Methods: - _get_page_content(): Private method to fetch the HTML content of the webpage. - scrape_element(selector, attribute=None): Scrapes elements based on the provided CSS selector. - - Example: - url = 'https://example.com' - scraper = HtmlScraper(url) - - # Scraping text content - titles = scraper.scrape_element('h2.title') - for title in titles: - print(f'Title: {title}') - - # Scraping attribute (e.g., href) content - links = scraper.scrape_element('a.link', attribute='href') - for link in links: - print(f'Link: {link}') - """ - - def __init__(self, url): - """ - Initializes the HtmlScraper instance. - - Args: - url (str): The URL of the webpage to scrape. - """ - self.url, self.sessions = url, None - self.requester = Requester() - self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'} - self.page_content = self._get_page_content() - - def _get_page_content(self,url=None, method:str='get', params=None, data=None, json=None, header=None, cookies=None, timeout=30, sessions=None, redirect=True, verify=True, proxy=None, ref=None, agent=None, pre_request=None): - """ - Private method to fetch the HTML content of the webpage. - - Returns: - str: The HTML content of the webpage. - - Raises: - Exception: If the request to fetch the page content fails. - """ - if url==None: - url=self.url - - if header == None: - header = self.headers - - if sessions!=None: - response, self.sessions = self.requester.requestSessions(url, method, params, data, json, header, cookies, timeout, sessions, redirect, verify, proxy, ref, agent, pre_request) - else: - response = self.requester.request(url, method, params, data, json, header, cookies, timeout, redirect, verify, proxy, ref, agent) - - if response.status_code == 200: return response.text - else: raise Exception(f"Failed to fetch page content. Status code: {response.status_code}") - - def scrape_element(self, selector, attribute=None): - """ - Scrapes elements based on the provided CSS selector. - - Args: - selector (str): The CSS selector to locate the desired elements. - attribute (str, optional): The attribute to extract from the elements (e.g., 'href' for links). Defaults to None. - - Returns: - list: A list of scraped content (either text or attribute values). - """ - soup = BeautifulSoup(self.page_content, 'html.parser') - elements = soup.select(selector) - - if attribute: - return [element.get(attribute) for element in elements] - else: - return [element.text.strip() for element in elements] - - def scrape_elements_grouped(self, info_dict): - """ - Scrapes elements based on the provided URL and element pathways, and groups the data. - - Args: - info_dict (dict): A dictionary containing the URL and element pathways. - - Returns: - dict: A dictionary with specified keys and values grouped in lists. - """ - if isinstance(info_dict, dict): - url = info_dict.get('url', '') - paths = info_dict.get('path', {}) - - if url: - self._get_page_content(url) - - if not paths: - raise ValueError("URL and element pathways must be specified in the info_dict.") - - grouped_data = {} - for key, path in paths.items(): - if isinstance(path, str): - grouped_data[key] = self.scrape_element(path) - elif isinstance(path, tuple): - grouped_data[key] = self.scrape_element(path[0], path[1]) - elif isinstance(path, dict): - grouped_data[key] = self.scrape_elements_grouped(key) - elif isinstance(path, list): - for i in path: - if isinstance(i, str): - grouped_data[key] = self.scrape_element(i) - elif isinstance(i, dict): - grouped_data[key] = self.scrape_elements_grouped(i) - else: raise ValueError('The path is either needs to be string, dict or a listof those.') - - return grouped_data - elif isinstance(info_dict, list): - return [self.scrape_elements_grouped(i) for i in info_dict] - else: return False - - -if __name__ == '__main__': - # url = 'https://example.com' - # scraper = HtmlScraper(url) - # titles = scraper.scrape_element('body > div:nth-child(1) > p') - # for title in titles: - # print(f'Title: {title}') - # links = scraper.scrape_element('a', attribute='href') - # for link in links: - # print(f'Link: {link}') - - - scrape_data = { - 'url':'https://example.com', - 'params': {'continue':False}, - 'path': { - 'title':{ - 'element':'body > div:nth-child(1) > p', - 'attr': None, - 'path': {} - }, - 'link' : { - 'element':'a', - 'attr': 'href', - 'path':{} - } - } - } \ No newline at end of file From 4805ad60e5b0402d6f5788e32d2b68e8cd93be57 Mon Sep 17 00:00:00 2001 From: Ranit Saha <162082159+coderooz@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:00:21 +0530 Subject: [PATCH 4/6] Add files via upload --- HtmlScraper.py | 182 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 HtmlScraper.py diff --git a/HtmlScraper.py b/HtmlScraper.py new file mode 100644 index 0000000..39b76a2 --- /dev/null +++ b/HtmlScraper.py @@ -0,0 +1,182 @@ +from Requester import Requester +from bs4 import BeautifulSoup +from FileHandler import write, read +from DataHandlers import get_unique + +class HtmlScraper: + + def __init__(self, url:str, setSessions:bool=False, set_header:bool=True, set_agent:bool=True, set_proxy:bool=False)->None: + """ + HtmlScraper + =========== + + This is a class that is to be used to scrape a website. + """ + self.url = url + self.setSessions, self.sessions = setSessions, None + self.req = Requester(set_header=set_header, set_agent=set_agent, set_proxy=set_proxy, agent_file='F:/Code Works/Python_works/storage/others/user-agent.txt') # proxy_file='F:/Code Works/Python_works/storage/others/proxies.txt') + self.souped = None + + def _rectiftyPathway(self, pathway): + + if isinstance(pathway, list): + pathway = [self._rectiftyPathway(p) for p in pathway] + elif isinstance(pathway, dict): + r = pathway.keys() + if 'tag' in r: + if 'attr' not in r: + pathway['attr'] = {} + if 'type' not in r: + pathway['type'] = 'find' + else: + pathway = {k: self._rectiftyPathway(v) for k,v in pathway.items()} + return pathway + + def _request(self, method:str='get', params:dict={}, ref:str='', response_code:int=200): + reqVals = {'url': self.url, 'method': method,'params': params, 'ref': ref, 'response_code': response_code} + req = None + if self.setSessions: + req, self.sessions = self.req.requestSessions(sessions=self.sessions, **reqVals) + else: + req = self.req.request(**reqVals) + return req + + def _souper(self, data, parser:str='html.parser'): + """ + _souper() + --------- + + Converts a html document data into BeautifulSoup class value. + """ + self.souped = BeautifulSoup(data, parser) + return self.souped + + def _getAtr(self, data, ty): + if isinstance(data, list): + return [self._getAtr(t, ty) for t in data if t is not None] + else: + r = None + if ty=='' or ty=='': + r = data.getText(strip=(True if ty=='' else False)) + else: + r = data.get(ty) + return r + + def _parser(self, selectorType:str='find', tagName='', attribute:dict={}, data=None)->(list|str|None): + """ + _parser() + --------- + This method is responsible for fetching the target element in the document. + + Parameters: + - `selectorType` str: The type of method that is to be used for fetching an element(s).Its values are + - find: Finds a single element, the first element that matches the values (tagName & attribute). Also the default value. + - find_all: Finds all the element of the same tag and atribute value. + - select_one: Similar to find, the tagName used would be the JS query selector value. Selecting this value returns a single value. + - select: Similar to find_all, the tagName used would be the JS query selector value. Selecting this value returns a list of value. + - `tagName` str: This parameter determines where to lookand what to fetch. Depending of on the `selectorType`, the value can be a tagName(for `find` & `find_all`) or a JS querySelctor value (for `select` or `select_one`) + - `attribbute` dict: This acts as a supporter for finding the target tag value. + - `data`: This parameter is to pass the html data where to look. Default is `None` which means, the page that will be parsed will be the page got during the requesting of the page. + + Returns: + - NoneType|list|str: Depending on the value passsed in `selectorType` parameter, the data type passed can be a list, str or a None type value. + - `select_one` or `find`: Return str + - `select` or `find_all`: Return list + - If no data found: Return None + """ + if data==None: + data=self.souped + try: + if selectorType == 'select': + k = data.select(tagName, attr=attribute) + elif selectorType == 'select_one': + k = data.select_one(tagName, attr=attribute) + elif selectorType == 'find': + k = data.find(tagName, attr=attribute) + elif selectorType == 'findall' or selectorType =='find_all': + k = data.find_all(tagName, attr=attribute) + return k + except: + return None + + # basic purpose + def storePage(self, fileName:str, data=None, seperator:str='\n', prevEmpty:bool=True)->None: + """ + storePage() + ----------- + This method is to store the page or the data in a file. + + Parameter: + - fileName str: Name of the file. + - data any: Takes the data that is to be inserted in the file. Default is `None`, which means the data stored will be the html data fetched during the request processes. + - seperator str: THis paramerter specifies, how the data points will be seperated in the file. Default is `\n` (a line break). + - prevEmpty bool: This parameter specifies if the existing data in the file should remain or be deleted. Default is `True`. Values:- + - `True`: The file will be emptied before inserting new data. + - `False`: The new data will be appended into the file with existing data. + """ + write(file_name=fileName, data=(self.souped if data==None else data), separator=seperator, emptyPervious=prevEmpty) + + # User use functions + def getAllUrls(self, data=None)->list[str]: + """ + getAllUrls() + ------------ + This method is fo getting all the urls in the parsed page + """ + return get_unique([i.get('href') for i in self._parser(selectorType='find_all',tagName='a', data=data)]) + + def getAllImages(self, data=None): + """ + getAllImages() + -------------- + Returns all the images in the page. + """ + imgs = {'tag': 'img', 'attr': {}, 'type': 'select', 'inner':{'imgLnk': 'src', 'alt':'alt'}} + return self.jsonParser(pathway=imgs, data=data) + + def getPageMeta(self, data=None): + """ + getPageMeta() + ------------- + Fetches the meta data of the page. + """ + pathway = {'title': {'tag':'title', 'get': ''}} + return self.jsonParser(pathway, data) + + def jsonParser(self, pathway:dict, data=None)->dict|None|list: + """ + jsonParser() + ------------ + This method is responsible for parsing the websitein the given structure. + """ + if data==None: + data = self._souper(self._request()) if self.souped==None else self.souped + + pathway = self._rectiftyPathway(pathway) + try: + if isinstance(pathway, str): + return self._parser(selectorType='select', tag=pathway, data=data) + elif isinstance(pathway, dict): + ret:dict = {} + if len(pathway) == 0: return None + if 'tag' in pathway.keys(): + k = self._parser(selectorType=pathway['type'], tagName=pathway['tag'], attribute=pathway['attr'], data=data) + if k is not None: + if 'get' in pathway.keys() and pathway['get'] != '' and pathway['get'] is not None: + ret['get'] = self._getAtr(ty=pathway['get'], data=k) + + if 'inner' in pathway.keys() and pathway['inner']!={}: + ret['inner'] = [self.jsonParser(data=n,pathway=pathway['inner']) for n in k] if isinstance(k, list) else self.jsonParser(data=k, pathway=pathway['inner']) + elif 'get' in ret.keys(): + return ret['get'] + else: return k + else: + for k,v in pathway.items(): + ret[k] = self._getAtr(ty=v, data=data) if isinstance(v, str) else self.jsonParser(pathway=v,data=data) + + return ret + elif isinstance(pathway, list): + return [self.jsonParser(pathway=i,data=data) for i in pathway] + except: + return None + From 21208fafee8960ed39d4e2ccf3649d6115dc5e42 Mon Sep 17 00:00:00 2001 From: Ranit Saha Date: Sat, 16 May 2026 07:16:20 +0530 Subject: [PATCH 5/6] chore: add repository setup files and documentation --- .editorconfig | 43 + .github/CODEOWNERS | 21 + .github/FUNDING.yml | 13 + .github/ISSUE_TEMPLATE/bug_report.md | 42 + .github/ISSUE_TEMPLATE/docs_improvement.md | 27 + .github/ISSUE_TEMPLATE/feature_request.md | 32 + .github/ISSUE_TEMPLATE/question.md | 26 + .github/PULL_REQUEST_TEMPLATE.md | 50 ++ .github/dependabot.yml | 33 + .github/labels.yml | 48 + .github/workflows/build.yml | 59 ++ .github/workflows/lint.yml | 47 + .github/workflows/test.yml | 49 + .gitignore | 158 ++++ .mcp-runtime.json | 10 + CHANGELOG.md | 76 ++ CODE_OF_CONDUCT.md | 136 +++ CONTRIBUTING.md | 308 +++++++ DataHandlers.py | 657 ++++++++++++++ DbHandler.py | 998 +++++++++++++++++++++ FileHandler.py | 288 ++++++ LICENSE | 9 + Readme.md | 479 ++++++++++ Requester.py | 440 +++++++++ SECURITY.md | 63 ++ __init__.py | 1 + custom-functions.project-mcp.json | 144 +++ pyproject.toml | 152 ++++ setup.py | 31 + 29 files changed, 4440 insertions(+) create mode 100644 .editorconfig create mode 100644 .github/CODEOWNERS create mode 100644 .github/FUNDING.yml create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/docs_improvement.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/question.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/dependabot.yml create mode 100644 .github/labels.yml create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 .mcp-runtime.json create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 DataHandlers.py create mode 100644 DbHandler.py create mode 100644 FileHandler.py create mode 100644 LICENSE create mode 100644 Readme.md create mode 100644 Requester.py create mode 100644 SECURITY.md create mode 100644 __init__.py create mode 100644 custom-functions.project-mcp.json create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..0fd43cd --- /dev/null +++ b/.editorconfig @@ -0,0 +1,43 @@ +# Editor configuration file +# https://editorconfig.org + +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +indent_size = 2 +trim_trailing_whitespace = false + +[*.yml] +indent_size = 2 + +[*.yaml] +indent_size = 2 + +[*.json] +indent_size = 2 + +[*.toml] +indent_size = 2 + +[*.cfg] +indent_size = 2 + +[*.ini] +indent_size = 2 + +[Makefile] +indent_style = tab + +[*.{bat,cmd,ps1}] +end_of_line = crlf + +[*.ps1] +indent_size = 2 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..73f45ab --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,21 @@ +# These are the users/teams that will be automatically added as reviewers to PRs +# that affect certain parts of the codebase. + +# Owner gets notified for all changes +* @Coderooz + +# Python files +*.py @Coderooz + +# Configuration files +*.yml @Coderooz +*.yaml @Coderooz +*.toml @Coderooz +*.cfg @Coderooz +*.ini @Coderooz + +# Documentation +*.md @Coderooz + +# GitHub workflows +.github/ @Coderooz diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..1a1e62f --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,13 @@ +# These are supported funding model platforms + +github: Coderooz +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +custom: ['https://coderooz.in/contact?subject=Sponsorship'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..f568189 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,42 @@ +--- +name: 🐛 Bug Report +about: Create a report to help us improve +title: '[BUG]: ' +labels: ['bug', 'needs-triage'] +assignees: '' +--- + +## Bug Description +A clear and concise description of what the bug is. + +## Steps To Reproduce +Steps to reproduce the behavior: +1. Import '...' +2. Call function '....' +3. See error + +## Expected Behavior +A clear and concise description of what you expected to happen. + +## Actual Behavior +What actually happened, including any error messages or unexpected output. + +## Code Sample +```python +# Minimal code snippet that reproduces the issue +``` + +## Environment +- **OS**: [e.g., Windows 10, Ubuntu 22.04, macOS 13] +- **Python Version**: [e.g., 3.12.0] +- **Package Version**: [e.g., 0.0.1] +- **Installation Method**: [e.g., pip, from source] + +## Additional Context +Add any other context about the problem here, such as: +- Screenshots +- Log output +- Related issues + +## Possible Solution +If you have ideas on how to fix this, please share them here. diff --git a/.github/ISSUE_TEMPLATE/docs_improvement.md b/.github/ISSUE_TEMPLATE/docs_improvement.md new file mode 100644 index 0000000..78ee5ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/docs_improvement.md @@ -0,0 +1,27 @@ +--- +name: 📝 Documentation Improvement +about: Suggest improvements to documentation +title: '[DOCS]: ' +labels: ['documentation', 'needs-triage'] +assignees: '' +--- + +## Documentation Issue +Describe what's missing, unclear, or incorrect in the current documentation. + +## Location +Where is the documentation issue located? +- [ ] README.md +- [ ] Function docstrings +- [ ] Code comments +- [ ] CONTRIBUTING.md +- [ ] Other (please specify) + +## Suggested Improvement +A clear and concise description of what should be added or changed. + +## Why This Matters +Explain why this documentation improvement would be valuable to users or contributors. + +## Additional Context +Add any other context or screenshots about the documentation issue here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..be604f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: ✨ Feature Request +about: Suggest an idea for this project +title: '[FEATURE]: ' +labels: ['enhancement', 'needs-triage'] +assignees: '' +--- + +## Problem Statement +Is your feature request related to a problem? Please describe. +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +## Proposed Solution +A clear and concise description of what you want to happen. + +## Alternative Solutions +A clear and concise description of any alternative solutions or features you've considered. + +## Use Case +Describe how you would use this feature and why it would be valuable. + +## Code Example (Optional) +```python +# Example of how you envision using this feature +``` + +## Additional Context +Add any other context, mockups, or screenshots about the feature request here. + +## Would You Like to Contribute? +- [ ] Yes, I would like to implement this feature +- [ ] No, I'm just suggesting the idea diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..1b957aa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,26 @@ +--- +name: 🔧 Question +about: Ask a question about this project +title: '[QUESTION]: ' +labels: ['question'] +assignees: '' +--- + +## Question +Your question here. Please be as specific as possible. + +## Context +What are you trying to accomplish? What have you tried so far? + +## Relevant Code +```python +# Include any relevant code snippets here +``` + +## Environment (if applicable) +- **OS**: +- **Python Version**: +- **Package Version**: + +## Additional Information +Any other details that might help answer your question. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..62c8e2d --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,50 @@ +## Description + + +## Type of Change + +- [ ] 🐛 Bug fix (non-breaking change which fixes an issue) +- [ ] ✨ New feature (non-breaking change which adds functionality) +- [ ] 💥 Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] 📝 Documentation update +- [ ] 🔧 Refactor (no functional changes) +- [ ] ⚡ Performance improvement +- [ ] 🧪 Test updates +- [ ] 🔒 Security fix + +## Related Issues + +Closes # + +## Changes Made + +- +- +- + +## Testing + +- [ ] I have added/updated tests that prove my fix is effective or that my feature works +- [ ] All existing tests pass locally with my changes +- [ ] I have tested this manually + +### Test Instructions +```bash +# Add instructions for testing your changes +``` + +## Checklist + +- [ ] My code follows the project's style guidelines +- [ ] I have performed a self-review of my code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my changes work +- [ ] New and existing unit tests pass locally with my changes + +## Screenshots (if applicable) + + +## Additional Notes + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..064913f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,33 @@ +version: 2 +updates: + # Python dependencies + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + timezone: "Asia/Kolkata" + open-pull-requests-limit: 10 + reviewers: + - "Coderooz" + labels: + - "dependencies" + - "python" + commit-message: + prefix: "deps" + include: "scope" + + # GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "09:00" + timezone: "Asia/Kolkata" + labels: + - "dependencies" + - "ci-cd" + commit-message: + prefix: "ci" diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 0000000..2426085 --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,48 @@ +- name: "bug" + color: "d73a4a" + description: "Something isn't working" +- name: "enhancement" + color: "a2eeef" + description: "New feature or request" +- name: "documentation" + color: "0075ca" + description: "Improvements or additions to documentation" +- name: "good first issue" + color: "7057ff" + description: "Good for newcomers" +- name: "help wanted" + color: "008672" + description: "Extra attention is needed" +- name: "question" + color: "d876e3" + description: "Further information is requested" +- name: "wontfix" + color: "ffffff" + description: "This will not be worked on" +- name: "duplicate" + color: "cfd3d7" + description: "This issue or pull request already exists" +- name: "needs-triage" + color: "fbca04" + description: "Needs review and triaging" +- name: "in-progress" + color: "0e8a16" + description: "Work is currently being done on this" +- name: "testing" + color: "c2e0c6" + description: "Related to testing" +- name: "performance" + color: "016175" + description: "Performance improvements" +- name: "refactor" + color: "1d76db" + description: "Code refactoring" +- name: "security" + color: "ee0701" + description: "Security-related issues or fixes" +- name: "dependencies" + color: "0366d6" + description: "Pull requests that update a dependency file" +- name: "python" + color: "2b5797" + description: "Python-related issues" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..6569245 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,59 @@ +name: Build and Package + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master, develop ] + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build wheel setuptools twine + + - name: Build package + run: | + python -m build + + - name: Check package + run: | + twine check dist/* + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: dist-packages + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + if: github.event_name == 'release' && github.event.action == 'published' + + steps: + - uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: dist-packages + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..8be634b --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,47 @@ +name: Python Lint + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master, develop ] + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.12', '3.13'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 black isort mypy + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Check formatting with Black + run: | + black --check --diff . + + - name: Check imports with isort + run: | + isort --check-only --diff . + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=120 --statistics + + - name: Type check with mypy + run: | + mypy --ignore-missing-imports . || true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..09887f1 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,49 @@ +name: Python Tests + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master, develop ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.12', '3.13'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pytest-cov + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install -e ".[dev]" + + - name: Run tests with pytest + run: | + pytest --cov=custom_functions --cov-report=xml --cov-report=html + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-report-py${{ matrix.python-version }} + path: htmlcov/ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..972ead7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,158 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# PEP 582 +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +.idea/ + +# VS Code +.vscode/ + +# Thumbs.db (Windows) +Thumbs.db + +# OS files +.DS_Store + +# Database files (project-specific) +*.db +*.sqlite + +# Test data +test_*.txt +test_*.csv +test_*.xlsx +test_*.pdf + +# Temporary files +*.tmp +*.temp +~$*.doc* +~$*.xls* diff --git a/.mcp-runtime.json b/.mcp-runtime.json new file mode 100644 index 0000000..6cad487 --- /dev/null +++ b/.mcp-runtime.json @@ -0,0 +1,10 @@ +{ + "port": 47000, + "pid": 8736, + "startedAt": 1778893041824, + "hostname": "Coderooz_PC", + "status": "running", + "project": "local-mcp-server", + "signature": "82660ef6-40f5-451a-9538-951e3ace798b", + "lastUpdated": "2026-05-16T01:25:16.828Z" +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1f1424b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,76 @@ +# CHANGELOG + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- Initial release of Custom Functions package +- `DataHandler` class for data manipulation utilities +- `DbHandler` class for SQLite database operations +- `FileHandler` class for file I/O operations +- `Requester` class for HTTP requests + +### Changed +- N/A + +### Deprecated +- N/A + +### Removed +- N/A + +### Fixed +- N/A + +### Security +- N/A + +--- + +## [0.0.1] - 2024-XX-XX + +### Added +- Initial project setup +- Core handler classes: + - `DataHandler`: Timestamp formatting and data utilities + - `DbHandler`: SQLite database CRUD operations, table management, and schema modifications + - `FileHandler`: File reading/writing, CSV/Excel/PDF handling + - `Requester`: HTTP request utilities +- Package distribution via `setup.py` +- MIT License + +--- + +## Template for Future Releases + +Copy and paste this template for new releases: + +## [VERSION] - DATE + +### Added +- + +### Changed +- + +### Deprecated +- + +### Removed +- + +### Fixed +- + +### Security +- + +--- + +**Author**: Ranit Saha +**Website**: [https://coderooz.in](https://coderooz.in) +**Contact**: [contact@coderooz.in](mailto:contact@coderooz.in) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..c186ff5 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,136 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[contact@coderooz.in](mailto:contact@coderooz.in) or via the +[contact form](https://coderooz.in/contact?subject=[Code_of_Conduct_Violation]). + +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. + +--- + +**Project**: Custom Functions +**Author**: Ranit Saha +**Website**: [https://coderooz.in](https://coderooz.in) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..38b3fb4 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,308 @@ +# Contributing to Custom Functions + +Thank you for your interest in contributing to **Custom Functions**! 🎉 + +This document provides guidelines and instructions for contributing. Please take a moment to read through it before making your first contribution. + +## Table of Contents + +- [Code of Conduct](#code-of-conduct) +- [Getting Started](#getting-started) +- [Development Setup](#development-setup) +- [How to Contribute](#how-to-contribute) +- [Pull Request Process](#pull-request-process) +- [Coding Standards](#coding-standards) +- [Testing Guidelines](#testing-guidelines) +- [Documentation](#documentation) +- [Commit Messages](#commit-messages) +- [Issue Reporting](#issue-reporting) +- [Community](#community) + +## Code of Conduct + +By participating in this project, you agree to abide by our [Code of Conduct](CODE_OF_CONDUCT.md). Please read it before contributing. + +## Getting Started + +### Prerequisites + +- Python 3.12 or higher +- pip (Python package manager) +- Git + +### Development Setup + +1. **Fork the Repository** + - Click the "Fork" button at the top right of the repository page + - Clone your fork locally: + ```bash + git clone https://github.com/YOUR_USERNAME/My_simple_functions.git + cd My_simple_functions + ``` + +2. **Set Up Remote Upstream** + ```bash + git remote add upstream https://github.com/coderooz/My_simple_functions.git + ``` + +3. **Create a Virtual Environment** + ```bash + python -m venv venv + source venv/bin/activate # On Windows: venv\Scripts\activate + ``` + +4. **Install Dependencies** + ```bash + pip install -e ".[dev]" + pip install flake8 black isort pytest pytest-cov + ``` + +5. **Create a Branch** + ```bash + git checkout -b feature/your-feature-name + # or + git checkout -b fix/your-bug-fix + ``` + +## How to Contribute + +### Types of Contributions + +We welcome various types of contributions: + +- 🐛 **Bug Fixes**: Fix issues reported in the issue tracker +- ✨ **New Features**: Add new functionality or handlers +- 📝 **Documentation**: Improve README, docstrings, or comments +- 🧪 **Tests**: Add or improve test coverage +- ⚡ **Performance**: Optimize existing code +- 🔧 **Refactoring**: Improve code structure without changing behavior +- 🔒 **Security**: Fix security vulnerabilities + +### Finding Issues to Work On + +- Look for issues labeled [`good first issue`](https://github.com/coderooz/My_simple_functions/labels/good%20first%20issue) for beginner-friendly tasks +- Check [`help wanted`](https://github.com/coderooz/My_simple_functions/labels/help%20wanted) for areas where we need assistance +- Always comment on an issue before starting work to avoid duplicate efforts + +## Pull Request Process + +1. **Ensure your code follows the coding standards** (see below) +2. **Add or update tests** for your changes +3. **Update documentation** if necessary +4. **Run the test suite** and ensure all tests pass +5. **Update CHANGELOG.md** with your changes +6. **Submit a Pull Request** using the [PR template](.github/PULL_REQUEST_TEMPLATE.md) + +### PR Checklist + +- [ ] Code follows project style guidelines +- [ ] Self-review completed +- [ ] Code is commented where necessary +- [ ] Documentation updated +- [ ] No new warnings generated +- [ ] Tests added/updated and passing +- [ ] CHANGELOG.md updated + +### Review Process + +- Project maintainers will review your PR +- Address any requested changes +- Once approved, a maintainer will merge your PR + +## Coding Standards + +We use the following tools to maintain code quality: + +- **Black**: Code formatter +- **isort**: Import sorter +- **flake8**: Linter +- **mypy**: Type checker (optional) + +### Running Formatters and Linters + +```bash +# Format code with Black +black . + +# Sort imports with isort +isort . + +# Run flake8 linter +flake8 . + +# Run all checks +black . && isort . && flake8 . +``` + +### Python Style Guidelines + +- Follow [PEP 8](https://peps.python.org/pep-0008/) style guide +- Use type hints where possible +- Write descriptive variable and function names +- Keep functions focused and single-purpose +- Maximum line length: 120 characters + +### Example Function Structure + +```python +def example_function(param1: str, param2: int = 10) -> dict: + """Brief description of what the function does. + + Args: + param1: Description of param1 + param2: Description of param2 (default: 10) + + Returns: + Description of return value + + Raises: + ValueError: When param1 is empty + """ + if not param1: + raise ValueError("param1 cannot be empty") + + # Implementation + result = {"key": param1, "count": param2} + return result +``` + +## Testing Guidelines + +### Running Tests + +```bash +# Run all tests +pytest + +# Run with coverage +pytest --cov=custom_functions --cov-report=html + +# Run specific test file +pytest tests/test_datahandler.py + +# Run with verbose output +pytest -v +``` + +### Writing Tests + +- Place tests in the `tests/` directory +- Name test files as `test_.py` +- Use descriptive test function names: `test__` +- Test both normal cases and edge cases +- Mock external dependencies when appropriate + +### Example Test + +```python +import pytest +from custom_functions.DataHandler import DataHandler + +def test_timestamp_returns_string(): + """Test that timestamp returns a formatted string.""" + result = DataHandler.timestamp() + assert isinstance(result, str) + +def test_timestamp_custom_format(): + """Test timestamp with custom format.""" + result = DataHandler.timestamp(format="%Y-%m-%d") + assert len(result) == 10 +``` + +## Documentation + +Good documentation is essential. When contributing: + +- Update README.md if adding new features +- Add docstrings to all public functions and classes +- Update inline comments for complex logic +- Include usage examples in docstrings + +### Docstring Format + +We use Google-style docstrings: + +```python +def my_function(param1: str) -> bool: + """One-line summary. + + Detailed description of what the function does, + its behavior, and any important notes. + + Args: + param1: Description of parameter + + Returns: + Description of return value + + Raises: + ExceptionType: When condition occurs + """ +``` + +## Commit Messages + +We follow [Conventional Commits](https://www.conventionalcommits.org/) specification: + +``` +(): + +[optional body] + +[optional footer(s)] +``` + +### Types + +- `feat`: New feature +- `fix`: Bug fix +- `docs`: Documentation changes +- `style`: Code style changes (formatting, etc.) +- `refactor`: Code refactoring +- `test`: Adding or updating tests +- `chore`: Maintenance tasks + +### Examples + +``` +feat(DataHandler): add JSON parsing utility +fix(FileHandler): handle empty file edge case +docs: update README with new installation instructions +test(DbHandler): add tests for createTb method +``` + +## Issue Reporting + +### Before Creating an Issue + +1. Check existing issues to avoid duplicates +2. Use the appropriate issue template +3. Provide as much context as possible + +### Issue Templates + +- 🐛 [Bug Report](.github/ISSUE_TEMPLATE/bug_report.md) +- ✨ [Feature Request](.github/ISSUE_TEMPLATE/feature_request.md) +- 📝 [Documentation Improvement](.github/ISSUE_TEMPLATE/docs_improvement.md) +- 🔧 [Question](.github/ISSUE_TEMPLATE/question.md) + +## Community + +- **Website**: [https://coderooz.in](https://coderooz.in) +- **Contact**: [https://coderooz.in/contact](https://coderooz.in/contact?subject=[message_subject]&message=[message]) +- **Email**: [contact@coderooz.in](mailto:contact@coderooz.in) + +## Recognition + +All contributors will be recognized in: + +- The CHANGELOG.md +- The README.md contributors section +- GitHub's contributors page + +Thank you for contributing to Custom Functions! 🙏 + +--- + +**Author**: Ranit Saha +**Website**: [https://coderooz.in](https://coderooz.in) diff --git a/DataHandlers.py b/DataHandlers.py new file mode 100644 index 0000000..018bf38 --- /dev/null +++ b/DataHandlers.py @@ -0,0 +1,657 @@ +from typing import Dict, List, Optional, Union +import time, json, os, pytz, re +from dateutil.relativedelta import relativedelta +from datetime import datetime, time, timedelta +from collections.abc import KeysView, ValuesView + +#### For Time Based Methods #### + + +def generate_time_intervals(start_time, duration_split:int=1, duration_format:str='d', output_format:str='human', datetime_format="%Y-%m-%d %H:%M:%S", end_time=None, excludeDate=None): + """ + Generate a list of time intervals based on the provided parameters. + + Parameters: + - start_time (str|datetime): The starting point for generating time intervals. + - duration_split (int): The duration of each time interval, specified in a numerical value. Default value is `1`. + - duration_format (str): A single-character code representing the format of the duration. + Options include 's' (seconds), 'm' (minutes), 'h' (hours), + 'd' (days), 'M' (months), and 'y' (years). + Default value is `d`. + - output_format (str): The desired output format for the time intervals. Options are 'unix' or 'human'. Default value is `human`. + - datetime_format (str, optional): A string specifying the format for human-readable time intervals. + Default is "%Y-%m-%d %H:%M:%S". + - end_time (datetime, str, optional): The end point for generating time intervals. + If not provided, the default is the current time. + - excludeDate (str, datetime, list[str], optional): The date's that are to be excluded. + + Returns: + - time_intervals (list): A list of generated time intervals based on the provided parameters. + """ + formats = {'s': 'seconds', 'm': 'minutes', 'h': 'hours', 'd': 'days', 'M': 'months', 'y': 'years'} + if isinstance(start_time, str): + start_time = datetime.strptime(start_time, datetime_format) + elif isinstance(start_time, (int, float, tuple, KeysView, ValuesView, dict, list)): + raise ValueError('The given format of data is invalid. Only excepts the datetime format or a str(i.e. 20-02-2020) whose format should match with `datetime_format` param.') + + delta = relativedelta(months=duration_split) if duration_format == 'M' else timedelta(**{formats[duration_format]: duration_split}) + + current_time, time_intervals = start_time, [] + if end_time is None: + end_time = datetime.now() + + while current_time <= end_time: + if output_format == 'unix': + time_intervals.append(current_time.timestamp()) + elif output_format == 'human': + time_intervals.append(current_time.strftime(datetime_format)) + else: + raise ValueError("Invalid output format. Choose between 'unix' or 'human'.") + current_time += delta + + if excludeDate!=None: + if isinstance(excludeDate, str): + excludeDate = [excludeDate] + elif isinstance(excludeDate, datetime): + excludeDate = [excludeDate.strftime(datetime_format)] + time_intervals = [i for i in time_intervals if i not in excludeDate] + return time_intervals + +def timestamp(given_time, format= "%Y-%m-%d %H:%M:%S", time_zone=None, normalize:str='sec'): + """ + timestamp() + ----------- + Returns the date and time in the specified format. + + Args: + + """ + + if isinstance(given_time, list): + return [timestamp(i, format, time_zone, normalize) for i in given_time] + elif isinstance(given_time, (int, float)): + given_time = convert_timestamp(given_time, normalize) + if time_zone is not None: + os.environ['TZ'] = time_zone + time.tzset() + return time.strftime(format, time.localtime(given_time)) + else: raise ValueError('The parameter passed in given_time is invalid. Please provide a valid data') + +def convert_timestamp(timestamp_ms, normalization='sec'): + """ Converts the unix timestamp to the desired format like second, minute or hour.""" + ty = type(timestamp_ms) + if ty == int or ty == float: + normalization_levels = {'millisecond': 1,'second': 1000,'minute': 1000 * 60,'hour': 1000 * 60 * 60,'day': 1000 * 60 * 60 * 24} + normalization_factor = normalization_levels.get(normalization, 1) + return timestamp_ms / normalization_factor + elif ty == list: return [convert_timestamp(i,normalization) for i in timestamp_ms] + else: raise TypeError("Argument type passed is not valid.") + +def future_timestamp(interval, unit, time_zone='UTC'): + """ + Calculate a future Unix timestamp based on the provided interval and unit. + + This function calculates a Unix timestamp representing a point in the future + by adding the specified interval of time to the current moment. + + Parameters: + - interval (int): The duration of the interval in the specified time unit. + - unit (str): A string representing the time unit of the interval. + Options include 's' (seconds), 'm' (minutes), 'h' (hours), + 'd' (days), 'mo' (months), 'y' (years). + - time_zone (str, optional): The time zone for the calculation. Default is 'UTC'. + + Returns: + - future_timestamp (float): The Unix timestamp representing the future point in time. + """ + units = {'s': 'seconds','m': 'minutes','h': 'hours','d': 'days','mo':'month', 'yrs':'year'} + tz = pytz.timezone(time_zone) + unit = units.get(unit, unit) # default to the input value if not found + future_timestamp = datetime.now(tz) + timedelta(**{unit: interval}) + return future_timestamp.timestamp() + +def split_interval(interval): + parts = re.findall(r'(\d+)([smhd])', str(interval)) + if parts: + value, unit = parts[0] + return int(value), unit + else: + raise ValueError(f'Invalid interval: {interval}') + +def to_unix_timestamp(date_time, formattype='%Y-%m-%d %H:%M:%S', time_zone='UTC'): + """Convert a human-readable date and time into the Unix timestamp in the specified time zone.""" + if isinstance(date_time, str): + dt =datetime.strptime(date_time,formattype) + tz = pytz.timezone(time_zone) + dt_localized = tz.localize(dt) + unix_timestamp = dt_localized.timestamp() + return unix_timestamp + elif isinstance(date_time, list): + return [to_unix_timestamp(i, formattype, time_zone) for i in date_time] + +def to_human_readable(unix_timestamp, formattype='%Y-%m-%d %H:%M:%S', time_zone='UTC'): + """Convert a Unix timestamp into a human-readable date and time in the specified time zone and format.""" + if isinstance(unix_timestamp, (int, float)): + dt_object = datetime.fromtimestamp(unix_timestamp) + tz = pytz.timezone(time_zone) + dt_localized = tz.localize(dt_object) + human_readable_timestamp = dt_localized.strftime(formattype) + return human_readable_timestamp + elif isinstance(unix_timestamp, list): + return [to_human_readable(i, formattype, time_zone) for i in unix_timestamp] + +def get_previous_day(date:str='', format:str='%d-%m-%Y', numberof_days:int=1): + """ + get_previous_day() + ------------------- + + This method is to get the previous date of the given date. + + Parameter: + - date (str, optional): This param takes the date who's previous date is to be fetched. Default is '', and will take the current date. + - format (str): This parameter takes the format of the date passed and will return the date inthe same format. Defaults to '%d-%m-%Y'. + - numberof_days int: takes the number of days previous to the given date's date is to be fetched. Defaault is 1. + + """ + date = datetime.now() if date == '' else dateFormat(date, format) + date = date - timedelta(days=numberof_days) + return date.strftime(format) + +def identify_date_format(date_str:str): + """ + identify_date_format() + ---------------------- + Identifties the date method. + """ + days_format = ['%Y-%m-%d','%m-%d-%Y','%d-%m-%Y','%Y/%m/%d','%m/%d/%Y','%d/%m/%Y','%Y.%m.%d','%m.%d.%Y','%d.%m.%Y','%Y %m %d','%m %d %Y','%d %m %Y'] + hours_format = ["%H:%M:%S","%H:%M","%I:%M:%S %p","%I:%M %p","%#H:%#M:%#S","%#I:%#M:%#S %p","%H:%M:%S.%f","%I:%M:%S.%f %p","%H:%M:%S %z","%I:%M:%S %p %z"] + formats = days_format + [f"{i} {h}" for i in days_format for h in hours_format] + + for fmt in formats: + try: + datetime.strptime(date_str, fmt) + return fmt + except ValueError: + pass + + return None + +def dateFormat(date:str, format1:str, format2:str = '')->(str|datetime): + """ + dateFormat() + ------------ + Changes the date format. And if the second format is provided then the date will be converted into the second provided format. + Parameter: + - `date` str: The date in string format. Ex: `20/07/2021`; + - `format1` str: The format in which the date is passed. Ex: `%d/%m/%Y`. + - `format2` (str, optional): This is the parameter used to change the given date time into the desired time format. Ex: `%Y/%m/%d`. + """ + try: + date:datetime = datetime.strptime(date, format1) + except: + t = identify_date_format(date) + if t == None: return date + date:datetime = datetime.strptime(date, t) + return date if format2=='' else date.strftime(format2) + + +#### For Dict Based Methods #### + +def equalizer_dict(data:List[dict], value='')->list: + """ + Make dictionaries equal by filling in missing keys and values. + + :param data: A list of dictionaries with varying keys and values. + :type data: List[dict] + + :param value: The default value to fill in missing positions in the dictionaries. + Defaults to an empty string (''). + :type value: Any, optional + + :return: A dictionary with keys present in all input dictionaries, + and values filled or padded according to the specified default value. + :rtype: list + """ + data_len = len(data) + data2 = {k:[] for k in data[0].keys()} + for i in range(data_len): + for k, v in data[i].items(): + ky = data2.keys() + if k not in ky: + if type(v) == type(value): + data2[k] = [value] * i + elif isinstance(v, (list, dict)): + data2[k] = [[]] * i + elif isinstance(v, str): + data2[k] = [''] * i + elif isinstance(v, (int, float)): + data2[k] = [0] * i + data2[k].append(v) + + for t in ky - set(data[i].keys()): + lo = data2[t][i - 1] + if type(lo) == type(value): data2[t].append(value) + elif isinstance(lo, (list, dict)): data2[t].append([]) + elif isinstance(lo, str): data2[t].append('') + elif isinstance(lo, (int, float)): data2[t].append(0) + return dlist_dict(data2) + +def dict_dimenstion_flatener(data:dict, catg): + """ + dict_dimenstion_flatener() + -------------------------- + This method is used to reducing a multi-dimention dict into a single dimention dict. + + Parameter: + - data (dict): takes the dict that is to made into a single dimention. + + """ + pass + +def dict_lister(data:list, opt:list=None)->dict: + """ + Converts a list of dictionaries with list-type values into a dictionary with lists. + + :param data: A list of dictionaries where each dictionary contains keys with list values. + :type data: list[dict] + + :param opt: An optional parameter specifying the keys to include in the resulting dictionary. + If not provided, it defaults to using all keys present in the first dictionary. + :type opt: list, optional + + :return: A dictionary where keys are from the 'opt' parameter (or all keys if 'opt' is not provided), + and values are lists containing corresponding values from the original dictionaries. + :rtype: dict + """ + t ={} + if opt == None: + opt = data[0].keys() + for i in range(len(data)): + for k,v in data[i].items(): + if k in opt: + if (k not in t): + t[k] = [None for _ in range(i)] + [v] if i > 0 else [v] + else: t[k].append(v) + if len(opt) == 1 and len(opt)!=None: return t[opt[0]] + return t + +def dict_filter(data, filter:list, catg=1)->dict[any]|list[dict[any]]: + """ + This function checks if a parameter exists in the dictionay. + :param data (dict|list[dict]): This takes the dict that is to be checked for the data presence. + :param filter list: This takes the list of values that is to be checked, if exists in the dict. + :praram catg int: This param to to set where to check the values for. either it is in the calues section of in the keys secton. Values accepted are + - `0`: This will check if a keys value is there, and if matches then the key is not added. + - `1`: This will return a dict with key names passed. + - `2`: This will return a dict without the key names passed. + """ + if isinstance(data, list): + return [dict_filter(data=d, filter=filter,catg=catg) for d in data] + elif isinstance(data, dict): + dc = {} + if catg == 1: + dc = {key: val for key, val in data.items() if key in filter} + elif catg == 0: + dc = {key: val for key, val in data.items() if val not in filter} + elif catg == 2: + dc = {key: val for key, val in data.items() if key not in filter} + return dc + else: raise TypeError('Only accepts a dict or list[dict].') + +def flatten_dict(d, parent_key='', sep='_', catg:int=0): + """ + This function flatens the dictionary into 1d i.e convertes the dictionay like {'a':{'b':'c'}} to {'a_b':'c'}. + :param d dict: This parameter takes the dict. + + """ + items = [] + if catg == 0: + for k, v in d.items(): + new_key = f'{parent_key}{sep}{k}' if parent_key else k + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + elif isinstance(v, list): + for i in v: + if isinstance(i, dict): + items.extend(flatten_dict(i, new_key, sep=sep).items()) + else: + items.append((new_key, i)) + else: + items.append((new_key, v)) + return dict(items) + elif catg == 1: + pass + else: raise ValueError('The value passed into the parameter `catg` is not accepted. The only value accepted are `0` & `1`.') + +def dict_reoganize(data, pattern:list): + """ + dict_reoganize() + ---------------- + + This method is to reorganze the keys of the given dict. + + Args: + data (dict|list): This patameter either takes the dict or a list of dict with similar keys. + pattern (list): This parameter takes the pattern in which the dict keys are to be arranged. + + Raises: + ValueError: The data passed in the data parameter is invalid. The paramerter only accepts either dict or a list of dict. + ValueError: _description_ + + Returns: + dict|list: This method either return the reorganized dict or the list of dict reoganized. + """ + if isinstance(data, list): + return [dict_reoganize(i, pattern) for i in data if isinstance(i, dict)] + elif isinstance(data, dict): + return {key: data[key] for key in pattern} + else: + raise ValueError('The data passed in the data parameter is invalid. The paramerter only accepts either dict or a list of dict.') + +#### For List Based Methods #### + +def dlist_dict(my_dict:dict, keys:list=[]) -> list: + """ + + Convert a dictionary into a list of dictionaries. + + This function takes a dictionary and converts it into a list of dictionaries. Each dictionary in the resulting list corresponds to a set of values for selected keys from the original dictionary. + + Parameters: + ----------- + my_dict : dict + The dictionary to be converted into a list of dictionaries. + Example: {'a': ['a_v1', 'a_v2'], 'b': ['b_v1', 'b_v2']} + + keys : list, optional + Optional parameter to select specific keys to include in the resulting list. + Defaults to an empty list ([]). + + Returns: + -------- + list + A list of dictionaries where each dictionary corresponds to a set of values for selected keys. + + Example: + -------- + >>> my_dict = {'a': ['a_v1', 'a_v2'], 'b': ['b_v1', 'b_v2']} + >>> result = dlist_dict(my_dict) + >>> print(result) + # Output: [{'a': 'a_v1', 'b': 'b_v1'}, {'a': 'a_v2', 'b': 'b_v2'}] + + >>> selected_keys = ['a'] + >>> result_selected_keys = dlist_dict(my_dict, keys=selected_keys) + >>> print(result_selected_keys) + # Output: [{'a': 'a_v1'}, {'a': 'a_v2'}] + + Notes: + ------ + - If `keys` parameter is not provided, all keys from the original dictionary will be included in the resulting list of dictionaries. + - The order of dictionaries in the resulting list is determined by the order of values for the first key in the original dictionary. + """ + + keys = my_dict.keys() if keys==[] else keys + result_list = [{k: v[i] for k, v in my_dict.items() if k in keys} for i in range(len(list(my_dict.values())[0]))] + return result_list + +def list_dlist(data, keys): + """ + This method list_dlist arranged the list list i.e.([['a','n'],['b','o']) into {key1:['a','b'], key2:['n','o']} + Note:- The inter list length and the length of strings list given must be equal. + :param list data: Takes the list that is to be arranged int the dict list format. + :param list keys: Thakes the list of strings that are to be used as the dictionary keys for the arrangement. + :return [dict] + """ + dict_key = {} + if isinstance(data[0], list) and len(keys) == len(data[0]): + for quote in data: + for i, value in enumerate(quote): + dict_key.setdefault(keys[i], []).append(value) + return dict_key + +#### For other Methods #### + +def get_unique(data:list, preserve:bool=False): + """Gets the unique data of the given list. It also has the festure fo arranging the data in a assendng ordre or sorting the given data out.""" + if preserve: + r = [] + for d in data: + if d not in r: + r.append(d) + return r + else: return sorted(list(set(data))) + +def add_index(data, index_id:int=0, index_name:str='index'): + """ + The `add_index` method is to add an id/index to the list data. + Parameter:- + - data(list): Takes the list of data which is to be indexed. The data in the list must be in a `dict` format. + - index_id (int|None): This param is note from where the indexing number should start after. Default value: 0, No will start from 1. + - index_name (str): The name of the key used to assign the index value. + """ + if len(data) <= index_id: raise ValueError('The value given in param index_id is invalid.') + for i in range(index_id+1, len(data)+1): + data[i-1][index_name] = i + return data + +def remove_empty_strings(data): + """Removes empty strings from a list or tuple.""" + return [string for string in data if string != ''] + +def decode_json(data): + """Attempts to decode a string as JSON.""" + try: + return json.loads(data) + except: + return None + +def json_parser(data:dict, pathway): + """ + This function is designed to parse data in a JSON/dictionary structure based on a specified pathway. + + Parameters: + - data (dict): The input JSON/dictionary data. + - pathway (str): The pathway specifying the keys to navigate the data. + + Returns: + The value at the specified pathway in the input data. + + Usage Examples: + 1. Simple pathway: + ```python + data = {'name': 'Alex', 'info': {'email': 'alex@gmail.com', 'age': 25}} + pathway = 'info > email' + result = json_parser(data, pathway) + print(result) # Output: 'alex@gmail.com' + ``` + + 2. Pathway with nested keys: + ```python + data = {'person': {'name': 'John', 'details': {'age': 30, 'city': 'New York'}}} + pathway = 'person > details > city' + result = json_parser(data, pathway) + print(result) # Output: 'New York' + ``` + + 3. Pathway with list indices: + ```python + data = {'people': [{'name': 'Alice'}, {'name': 'Bob'}]} + pathway = 'people > 1 > name' + result = json_parser(data, pathway) + print(result) # Output: 'Bob' + ``` + + 4. Using a list of pathways to extract multiple values: + ```python + data = {'user': {'name': 'Alex', 'email': 'alex@gmail.com', 'age': 25}} + pathways = ['user > name', 'user > email'] + result = json_parser(data, pathways) + print(result) # Output: {'name': 'Alex', 'email': 'alex@gmail.com'} + ``` + 5. Using Special keys: + ```python + data = {'user': { + 'profile': {'name': 'John','address': {'city': 'New York', 'country': 'USA'},}, + 'preferences': {'theme': 'dark', 'notifications': True}, + } + } + + pathway = { + '__pathway__': { + 'path': 'user > profile', + 'data': ['name', 'address > country'], + },'preferences': 'user > preferences', + } + + result = json_parser(data, pathway) + print(result) # {'name': 'John','address > country': 'USA','preferences': {'theme': 'dark', 'notifications': True}} + ``` + """ + if isinstance(pathway, list): + return {i.split('>')[0]:json_parser(data, i) for i in pathway} + elif isinstance(pathway, str): + k = data + if '>' in pathway: + path = pathway.split('>') + for pa, le in enumerate(path): + i = setNum(le.strip()) + if i == '`list`' and isinstance(k, list): + k = [json_parser(k, f"{str(n)} > {' > '.join(path[pa + 1:])}") for n in range(len(k))] + break + elif (isinstance(k, list) and isinstance(i, int) and len(k)-1 >= i) or (isinstance(k, dict) and i in k.keys()): + k = k[i] + else: + k = None + break + else: + k = k[pathway] + return k + elif isinstance(pathway, dict): + n = {} + for k,v in pathway.items(): + if k == '__pathway__': + if isinstance(v, dict): + if isinstance(v['data'], dict): + n = {**n, **{h:json_parser(data, str(v['path'] + f' > {i}')) for h,i in v['data'].items()}} + elif isinstance(v['data'], list): + n = {**n, **{i.split('>')[-1].strip():json_parser(data, v['path'] + f' > {i}') for i in v['data']}} + elif isinstance(v['data'], str): + n[k] = json_parser(data, v['data']) + elif isinstance(v, list): + n = {**n, **{i['path'].split('>')[-1].strip():json_parser(data, {'__pathway__': i}) for i in v}} + else: + n[k] = json_parser(data, v) + return n + else: raise ValueError('The pathway given is not acceptable/invalid. Please check the pathway.') + +def round_values(data): + """Rounds the values in a list, tuple, or dictionary if they are integers or floats.""" + data_type = type(data) + if data_type == list: + return [round(i) for i in data if isinstance(i, (float, int))] + elif data_type == dict: + return {key: round_values(value) for key, value in data.items()} + elif isinstance(data, (float, int)): + return round(data) + else: + print(f'The value passed is not compatible with the method. The type of value passed is {data_type}.\nPlease try again.') + +def check_difference(*lists, time_period= None): + max_length = max([len(list) for list in lists]) + time_period = time_period if (time_period!= None) else min([len(list) for list in lists]) + lists = [list + [0] * (max_length - len(list)) for list in lists] + direction = [] + for i in range(1, time_period): + diff = sum([list[i] for list in lists]) + prev_diff = sum([list[i-1] for list in lists]) + if diff > prev_diff: + direction.append(1) + elif diff < prev_diff: + direction.append(-1) + else: + direction.append(0) + return direction[:-time_period] + +def get_differences(list1, list2): + """ This function is used to get the differences beteween the two list of numbers of a list individually at the specific positions.""" + differences = [] + for i in range(len(list1)): + diff = list1[i] - list2[i] + differences.append(diff) + return differences + +def calculate_difference_percentage(num1, num2): + """This function calculates the difference in percentage between two given numbers.""" + per = [] + for i in range(len(num1)): + difference = num1[i] - num2[i] + per.append(difference / min(num1[i], num2[i]) * 100) + return per + +def check_value_exists(value, param): + """This method/function is used to determine wether a certan value exists or not in the value/data.""" + if isinstance(param, dict): return value in param.values() + elif isinstance(param, (set, str)): return value in param + else: return value in list(param) + +def valreplace(data, target:str, replace:str, keyTy:bool=False): + """ + This method/function is used for replaceing certain or wanted values in the given subject data. + + Paramerters: + ----------- + - data (str|list|dict): This + - target (str): This parameter takes the target that has to be changed. + - replace (str): This parameter takes the value that is to be replaced by the target. + - keyTy (bool): If the data provided is a key then this value, if set to `True` then will also check look into the keys of the dict and change it accordingly. + + Return: + ------- + - str | list | dict : This method will not change the value type given and will retuen the data in same type. + """ + if isinstance(data, dict): + data = {(replace if k == target else k): v for k, v in data.items()} if keyTy else {k:valreplace(v, target, replace) for k,v in data.items()} + return data + elif isinstance(data, (list, KeysView, ValuesView)): + return [valreplace(x, target, replace, keyTy) for x in data] + elif isinstance(data, str): + if keyTy: return ' '.join([word.replace(target, replace) if word == target else word for word in data.split()]) + else: return data.replace(target, replace) + else: raise ValueError(f'This method only expects list, dict or a string. Not {type(data)}') + +def space_remover(data): + """ + This method is usd for removing any spaces in a string that is in front or behind the string. + This method can work on strings , dicts and lists of string. + """ + if isinstance(data, dict): + return {k.strip(): space_remover(v) for k,v in data.items()} + elif isinstance(data, list): + return [space_remover(i) for i in data] + elif isinstance(data, str): + return data.strip() + else: return data + +def setNum(data): + """ + This metod is used for making any possible number or float that is in a string format turn into one. + """ + if isinstance(data, dict): return {k.strip(): setNum(v) for k,v in data.items()} + elif isinstance(data, list): return [setNum(i) for i in data] + elif isinstance(data, str): + try: + return int(data) + except ValueError: + try: + return float(data) + except ValueError: + return data + else: + return data + +def get_similarities(list1, list2): + """ + This function returns the similarity between two lists and returns the simialar. + """ + return list(set(list1).intersection(set(list2))) + diff --git a/DbHandler.py b/DbHandler.py new file mode 100644 index 0000000..e5eef9f --- /dev/null +++ b/DbHandler.py @@ -0,0 +1,998 @@ +import sqlite3, csv, json +import pandas as pd +from DataHandlers import valreplace, equalizer_dict +from FileHandler import getExtention, read, fileExists, write, write_csv +from collections.abc import KeysView, ValuesView +import mysql.connector as myqC +from datetime import datetime +from typing import List, Union, Dict + +class SqliteHandler(): + """ + The `DbSqliteHandler` class simplifies interactions with SQLite3 databases in Python, offering a dynamic and efficient approach. It is designed to accelerate the development of database-related projects using Python's SQLite3 module. + + Features: + - Insert Data (insert) + - Fetch Data (fetch) + - Display Data (displayData) + - Execute Custom SQL (execute) + - Get Table Names (getTb) + - Check Index Existence (checkIndex) + - Add Index (addIndex) + - Close Database Connection + + Initialization: + --------------- + To use `DbSqliteHandler`, create an instance by providing the database name (`dbname`) and an optional path to the database directory (`dbPath`). If no path is specified, the database will be created in the current working directory. + + Example: + ```python + db_handler = SqliteHandler("my_database.db", "path/to/database/directory") + ``` + + Insert Data (insert): + --------------------- + Insert data into a specified table using the `insert` method. Provide the table name (`table`), column names (`columns`), and a list of values to be inserted (`values`). + + Example: + ```python + db_handler.insert("my_table", "column1, column2", (value1, value2)) + ``` + + Fetch Data (fetch): + ------------------- + Retrieve data from a table with the `fetch` method. Specify the table name (`table`) and optionally, specific columns to retrieve (`columns`). You can also provide a custom SQL query for advanced retrieval. + + Example: + ```python + data = db_handler.fetch("my_table", "column1, column2", "column1 = 'some_value'") + ``` + + Execute Custom SQL (execute): + ----------------------------- + Execute custom SQL queries using the `execute` method. Provide the SQL code as the `data` parameter. Use `multi=True` for executing multiple statements within a single call. + + Example: + ```python + db_handler.execute("CREATE TABLE new_table (column1 TEXT, column2 INTEGER);", multi=True) + ``` + + Database Management: + -------------------- + `DbSqliteHandler` provides various methods for managing tables and indexes, including creating, renaming, cleaning, or deleting tables. + + Example (Creating a Table): + ```python + db_handler.createTb("new_table", ["column1 TEXT", "column2 INTEGER"]) + ``` + + Closing the Connection: + ----------------------- + To close all connections and end the session, call the `close_connection` method. + + Example: + ```python + db_handler.close_connection() + ``` + + `DbSqliteHandler` offers a flexible and efficient way to interact with SQLite3 databases in Python, simplifying database-related tasks and enhancing the productivity of your projects. + """ + + def __init__(self, dbname, dbPath:str='.', json_import:bool=False, default_timeout:int=5000): + """ + Initializes the DbSqliteHandler instance. + + Parameters: + - `dbname` (str): The name of the database. + - `dbPath` (str, optional): The path to the database directory. Default is None. + """ + self.db_init = None + self.db_conn = None + self.dbName = dbname + self.dbPath = dbPath + self.dbFullPath = self.dbPath+'/'+self.dbName + ext = getExtention(self.dbFullPath) + + if json_import and ext == 'json': + self.load_dbJson(self.dbFullPath, True) + elif ext == 'db': + self.db_init = sqlite3.connect(self.dbFullPath) + self.db_conn = self.db_init.cursor() + else: + raise TypeError('File type error! only excepts json file containing dbcreating data or the db path.') + + self.DbtimeOut(default_timeout) + + def execute(self, query, data:list=[], multi: bool = False, auto_commit=True): + """ + execute() + --------- + + Executes SQLite-related code. + + Parameters: + - `query` (str): The SQL code to execute. + - `data` (list): Takes the list of values. Default is []. + - `multi` (bool): True if executing multiple statements. Default is False. + - `auto_commit` (bool): True to commit changes automatically after execution. Default is True. + + Returns: + sqlite3.Cursor: The result of the executed query. + + If Error/Or exceptions: + Prints out exceptions and rolls back the transaction if auto_commit is True. + + """ + try: + if multi==True and len(data) > 0: + result = self.db_conn.executemany(query, data) + elif multi==False: + result = self.db_conn.execute(query) + else: + raise ValueError('The data/value query passed is not valied.') + if auto_commit: + self.db_init.commit() + return result + except sqlite3.Error as e: + print("SQLite error:", e) + if auto_commit: self.db_init.rollback() + return None + + # --- Data handleing --- # + + def insert(self, table:str, columns, values, createTb:bool=False): + """ + Inserts data into the specified table. + + Parameters: + - `table` (str): The name of the table to insert data into. + - `columns` (str, list, KeysView): Comma-separated column names. + - `values` (list): List of values to be inserted. + - `createTb` (bool): This will create a table in the db if not available. Default is `False`. + + Returns: + sqlite3.Cursor: The result of the executed query. + """ + if createTb==True and self.getTb(table_name=table) == False: self.createTb(tbName=table, columns=columns, primary_key='id') + k=False + if isinstance(columns, (KeysView, list, tuple, ValuesView)): + keys = ['INDEX','KEY','SELECT','INSERT','UPDATE','DELETE','FROM','WHERE','JOIN','INNER','LEFT','RIGHT','GROUP BY','ORDER BY','AS','COUNT','SUM','MAX','MIN','AVG','DISTINCT','AND','OR','NOT','BETWEEN','LIKE','IN','NULL','TRUE','FALSE','TOP','LIMIT','OFFSET'] + for k in keys: + columns = valreplace(columns, k, '_'+k.upper(), 1) # type: ignore + columns = valreplace(columns, k.lower(), '_'+k.lower(), 1) # type: ignore + columns = ','.join(columns) + + if isinstance(values, (KeysView, list, tuple, ValuesView)): + value = [] + for val in values: + if isinstance(val, (KeysView, list, tuple, ValuesView)): + value.append(str(tuple(val))) + + elif isinstance(val, str): + value = str(tuple(values)) + return self.execute(f'INSERT INTO {table} ({columns}) VALUES {value};') + value = ','.join(value) + k = self.execute(f'INSERT INTO {table} ({columns}) VALUES {value} ;') + elif isinstance(values, str): + k = self.execute(f'INSERT INTO {table} ({columns}) VALUES ({values});') + else: raise ValueError('Invalid input format! Please provide a valid set of values.') + + def json_insert(self, table_name:str, data, createTb:bool=False, ifExist:str='')->None: + ''' + JSON_INSERT() + ------------- + + This method is used to insert data into the table using json format data. + Parameters: + - `table_name` (str):This parametere of the method tales the name of the table in which the data is to inserted. + - `data` (list|dict): This parameter takes the data either in dict format or a list containing dicts if multiple entries are to add. + - `createTb` (bool): .. + - `ifExist` (list): A list of fields that should exist before inserting the record. If any field does not + - return None + ''' + + table_column = self.getColumnNames(table_name) + + if isinstance(data, dict): + col, query = [], [] + for k,i in data.items(): + if k in table_column and isinstance(i, (str, float, int)): + col.append(k) + query.append(i) + elif isinstance(data, list) and len(data) > 0: + data = equalizer_dict(data) + col = data[0].keys() + query = [list(i.values()) for i in data] + else:raise ValueError('The data type passed is invald. The data paramerter takes a dict or a list of dict.') + + col = list(col) + if ifExist!='': + check_data = self.fetch_unique(table_name, ifExist) + check_data = {ifExist: check_data} if isinstance(check_data, list) else check_data + for k,v in check_data.items(): + col_idx = col.index(k) + query = [val for val in query if val[col_idx] not in v] + + self.insert(table_name, col, query, createTb) + + def update(self, table:str, updatedata, condition:str=''): + """ + update() + -------- + + This method is to update data in the table + + Parameter: + - `table`: This parameter takes the name of the table. + - `updatedata`: This parameter takes the data that is to updated. + - `"ColumnName='data1' AND ColumnName2='data2'" OR {'ColumnName':'data1', 'ColumnName2':'data2'} + ` + - `condition`: This parameter is set where the data is to be updated. + - `id='5'` + Default is ''. If left empty, then the update will happen all over the column which is specified. + + Return: + - `bool`: This method returns a boolean. True is success or False for failed opeeration.s + + Usage Example: + ``` + # assumning cl is the class like. + updatedata = "user_email='alex123@gmail.com'" + condition = "name='Alex'" + cl.update('tableName', updatedata, condition) + ``` + """ + condition = f'WHERE {condition}' if condition!='' else '' + if isinstance(updatedata, dict): + updatedata = ', '.join(["{}='{}'".format(k,v) for k,v in updatedata.items()]) + t = self.execute(f'UPDATE {table} SET {updatedata} {condition};') + if t : return True + return False + + def fetch(self, table:str, columns:str='*', query: str|list[str]='', limit:int=0, Offset:int=0, fetchAll:bool=True, assc:str='', desc:str='', detailed:bool=True)->list[str, float, int]: + """ + fetch() + ------- + + Fetches data from the specified table based on the query. + + Parameters: + - `table` (str): The name of the table to fetch data from. + - `query` (str|list|dict|optional): The SQL query/Search parameter that is to be executed. + - `columns` (str): The columns that needs to be fetched. + - `limit` (int, optioanl): This parameter is to set the number of columns to fetch. + - `offset` (int, optional): The parameter if speciied will get the columns from the limit number of columns to the number specifed in this parameter. eg: from column 5 to 23. This parameter will only be in effect of the limit parameter is use. + - `assc` (str, optional): The columns that are to be fetched in ascending order. + - `desc` (str, optional): The columns that are to be fetched in descending order. + - `fetchAll` (bool, optional): The columns that needs to be fetched. + + Returns: + sqlite3.Row or list of sqlite3.Row: The fetched data. + + Usage Example: + ``` + # assumning cl is the class like. + columns = "column1, column2" + query = "column1 = 'some_value'" + desc = "column2" + data = cl.fetch("my_table", columns, query=query, fetchAll=True, desc=desc) + ``` + """ + + if self.getTb(table) == False: + raise ValueError(f'The table({table}) is not present in the database.') + + if isinstance(query, str) and query!='': + query = f' WHERE {query}' + elif isinstance(query, list)and len(query) > 0: + query = ' WHERE ' + ' AND '.join(query) + elif isinstance(query, dict)and len(query.keys()) > 0: + query = ' WHERE ' + ' AND '.join([f"{k}='{v}'" for k,v in query.items() if v!=None or v!='']) + + order:str = '' + if desc != '' or assc != '': + col:list= self.getColumnNames(table) + ord:list = [] + if assc!='' and assc in col: ord.append(f'{assc} ASC') + if desc!='' and desc in col: ord.append(f'{desc} DESC') + order = ' ORDER BY ' + ', '.join(ord) + + if limit > 0: + limit:str = f' LIMIT {str(limit)},{str(Offset)}' if Offset != 0 and Offset > limit else f' LIMIT {str(limit)}' + else: + limit:str = '' + query = f"SELECT {columns} FROM {table}{query}{order}{limit};" + ret = self.execute(query) + try: + if ret is not None: + if detailed==False: + if fetchAll: return ret.fetchall() + else: return ret.fetchone() + else: + col = [column[0] for column in ret.description] if ret.description else [] + if fetchAll: return [dict(zip(col, row)) for row in ret.fetchall()] + elif ret.fetchone() is not None: return dict(zip(col, ret.fetchone())) + else: return [] + except Exception as e: + print(e) + return [] + + def getTbData(self, table_name:str,columns:str='*',query:str='', limit:int=0, offset:int=0, fetchAll:bool=True, desc:str=''): + """ + Retrieves data from the specified table and returns it in a pandas DataFrame format. + + Parameters: + - `table_name` (str): The name of the table to fetch data from. + - `columns` (str, optional): The columns to retrieve. Default is '*'. + - `query` (str, optional): The SQL query or condition for data retrieval. Default is an empty string. + - `limit` (int, optional): This parameter takes the number of columns that are to be fetched. + - `offset` (int, optional): The parameter if speciied will get the columns from the limit number pf columns to the number specifed in this parameter. eg: from column 5 to 23. This parameter will only be in effect of the limit parameter is use. + - `fetchAll` (bool, optional): True to fetch all rows, False to fetch only the first row. Default is True. + - `desc` (str, optional): This parameter, if specified will get the data from the table in descending order by the name of the column mentioned. + Returns: + pd.DataFrame: A DataFrame containing the fetched data. + + Example: + ```python + data_frame = db_handler.getTbData("my_table", "column1, column2", "column1 = 'some_value'") + ``` + + Note: + This method is similar to the fetch function, but it returns the data in a pandas DataFrame format. + """ + if self.getCount(table_name) > 0: + data = self.fetch(table_name,columns, query, limit, offset, fetchAll, desc) + return pd.DataFrame(data, index=None) + else: + print(f'The table(`{table_name}`) is empty with no data.') + return False + + ### Table work/ altering related method. ### + + def rearrange_table(self, table:str, orderOf:str, orderBy:str='asc', backup:bool=True)->None: + """ + rearrange_table() + ----------------- + This method us to rearrange data inside the table. + Parameters: + - table str: The name of the table. + - orderOf str: The column by which the data is needed to be arranged. + - orderBy str: The order in which the column needed to + + """ + pass + + def fetch_unique(self, table:str, column:str)->List[str | int | float] | dict[str, List[str | int | float]]: + """ + fetch_unique() + -------------- + This method fetches the unique values of a column. + + Parameter: + - table str: Takes the name of the table. + - column str: takes the name of the columns whose unique data is to fetched. If more than one table's data is required then the column names must be seperated by a comma(,) for the function to work properly. + + Returns: + - List | Dict[List]: This method returns a list of unique values in the coloum. In case if more than one column is porvided then dict containing list as its values with column names as its key. + """ + column:list = column.split(',') + columnList:list = self.getColumnNames(table) + if len(column) > 1: + return {k: self.fetch_unique(table, k) for k in column if k in columnList} + elif len(column) == 1: + data = self.fetch(table, f'DISTINCT {column[0]}') + data = [i[column[0]] for i in data] if len(data) > 0 else [] + return data + else: return [] + + def load_dbJson(self, data=None, fileName:bool=False)->None: + """ + This method will create the database with all its tables table and values(if provided). + """ + if fileName and fileExists(data): + self.load_dbJson(read(data, decode_json=True)) + elif isinstance(data, (str, dict)) and fileName==False: + data = json.dumps(data) if isinstance(data, str) else data + self.dbName = data['db_name'] + self.db_init = sqlite3.connect(self.dbPath+'/'+self.dbName) + self.db_conn = self.db_init.cursor() + for table in data['tables']: + print(table['table_name']) + self.createTb(table['table_name'], table['column_names']) + if table['data'] != []: + self.insert(table['table_name'], table['column_names'], table['data']) + if table['indexs'] != []: + [self.addIndex(idx['index_name'], idx['cols']) for idx in table['indexs']] + else: + raise ValueError('Check the value given passed as arguments.') + + def export_data(self, catg:str='json', tableName:str=''): + if catg=='json': + data = {'db_name': self.dbName,'tables':[],'created_on': str(datetime.now().strftime('%d-%m-%Y %H:%M:%S %p'))} + for tb in self.getTb(): + k=self.get_info(tb) + if k['rows'] > 0: + k['data'] = self.fetch(tb, detailed=False) + data['tables'].append(k) + write(f'{self.dbPath}/{self.dbName.replace('.','_')}.json', data, emptyPervious=True) + elif catg=='xls': + pass + # write(f'{self.dbPath}/{self.dbName.replace('.','_')}.csv', data, emptyPervious=True) + elif catg == 'csv': + pass + else: raise ValueError('The type of file given is not accepted.') + + def getCount(self, table_name:str, columns:str='*', query:str='')->int: + """ + getCount: + ========= + + This mehtod is to count the number of the row present in the table according the the query. + + Args: + - `table_name` (str): Takes the name of the table. + - `columns` (str) : Takes the name of the column that is to be counted. + - `query` (str) : Takes the search query by which the table is to be counted. + + Return: + int : Returns the number of rows present n tahe tble according to the query. + """ + to = f'COUNT({columns})' + col = self.fetch(table_name, to, query, detailed=False) + if col != []: + try: + return int(col[0][0]) + except: + return col + return False + + def alterTb(self, tbName:str, queryType:str, modify): + """This method is to alter tables data. + + Args: + tbName (str): parameter takes the name of the table. + queryType (str): This parameter takes the data in string format is to specify where to alter. + modify (list): This parameter takes the data in list format is to specify what to alter with. + + Returns: + _type_: _description_ + """ + if isinstance(modify, list): + modify = str(','.join(modify)) + query =f'ALTER TABLE {tbName} {queryType.upper()} {modify};' + return self.execute(query) + + def csv_insert(self, table_name:str, csv_file_path:str): + """ + Creates a table (if it doesn't exist) and adds data from a CSV file. + + Parameters: + table_name (str): The name of the table to be created or used. + csv_file_path (str): The path to the CSV file containing data to be inserted into the table. + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + if not self.getTb(table_name): + with open(csv_file_path, 'r') as csvfile: + csv_reader = csv.reader(csvfile) + headers = next(csv_reader) + column_types = ['TEXT' for _ in headers] + columns = [f"{header} {column_type}" for header, column_type in zip(headers, column_types)] + self.createTb(table_name, columns) + + with open(csv_file_path, 'r') as csvfile: + csv_reader = csv.DictReader(csvfile) + csv_data = [row for row in csv_reader] + json_data = json.dumps(csv_data, indent=2) + return self.json_insert(table_name, json_data) + except Exception as e: + print(f"Error: {e}") + return False + + def get_excel(self, tbName:str='', columns:str='*', query:str='', fetchAll:bool=True, desc:str='', fileName:str='', filePath:str='.'): + """ + This method get the specified table and saves the data in the file + + Args: + tbName (str): _description_ + columns (str, optional): _description_. Defaults to '*'. + query (str, optional): _description_. Defaults to ''. + fetchAll (bool, optional): _description_. Defaults to True. + desc (str, optional): _description_. Defaults to ''. + fileName (str, optional): _description_. Defaults to ''. + filePath (str, optional): _description_. Defaults to './'. + + Returns: + file: Returns a saved file. + """ + try: + data = self.getTbData(tbName,columns,query,fetchAll, desc) + fileName = fileName if fileName!=None else f'{tbName}.csv' + fileName = f'{filePath}/{fileName}' + data.to_csv(fileName, index=False) + return 1 + + except Exception as e: + print(e) + return 0 + + def beginTransaction(self): + """Begin a transaction.""" + self.db_init.isolation_level = None + self.execute("BEGIN TRANSACTION;") + + def commitTransaction(self): + """Commit the current transaction.""" + self.execute("COMMIT;") + self.db_init.isolation_level = '' + + def rollbackTransaction(self): + """Roll back the current transaction.""" + self.execute("ROLLBACK;") + self.db_init.isolation_level = '' # Auto-commit mode is turned on + + def getColumnNames(self, table_name): + """ + Fetches the column names of a specified table. + + Parameters: + table_name (str): The name of the table. + + Returns: + list: A list of column names. + """ + query = f"PRAGMA table_info({table_name});" + result = self.execute(query) + columns = [row[1] for row in result.fetchall()] + return columns + + def get_info(self, table_name:str='')->dict: + """ + Retrieve information about a specific table in an SQLite database. + + Parameters: + table_name (str): The name of the table to retrieve information about. + + Returns: + dict or None: A dictionary containing table information, or None if an error occurs or the table does not exist. + + The returned dictionary contains the following keys: + - table_name: The name of the table or the entire Database. Default is ''. This means that it will give the info of the intire table + - table_description: A description of the table (if available). + - column_names: A list of column names in the table. + - column_types: A list of column data types corresponding to the column names. + """ + try: + if table_name == '': + return {i: self.get_info(i) for i in self.getTb()} + else: + columns_info = self.execute(f"PRAGMA table_info({table_name})").fetchall() + if not columns_info: return None + column_names = [info[1] for info in columns_info] + column_types = [info[2] for info in columns_info] + keyType = ['', 'PRIMARY KEY','SECONDARY KEY'] + keys = [keyType[int(info[5])] for info in columns_info] + return {'table_name': table_name,'column_names':[' '.join(k).strip() for k in zip(column_names, column_types, keys)],'rows': self.getCount(table_name), 'indexs':self.getIndexes(table_name)} + except sqlite3.Error as e: + print(f"Error: {e}") + + def getTb(self, table_name:str=None): + ''' + getTb() + ------- + + This method is to get the list of all the tables present in the database. + Additional: This method also has the function to check if the table exists or not. + + Parameter: + - `table_name` (str): This parameter takes the name of the table to look for. + + Return: + - If the table_name parameter is given the either it will be returned bool true or else false. If the table_name is not added then a list of tables will be returned. + ''' + result = self.execute("SELECT name FROM sqlite_master WHERE type='table';") + data = [row[0] for row in result.fetchall()] + if table_name == None:return data + elif table_name != None and table_name in data: return True + else: return False + + def DbtimeOut(self, timeout:int=0): + if timeout == 0: + timeout = self.default_timeout + else: + self.default_timeout=timeout + + self.execute(f'PRAGMA busy_timeout = {timeout};') + + def getIndex(self, tbName:str, idxName:str): + """Check if the index exists in the table.""" + result = self.execute(f"PRAGMA index_info({idxName});") + return len(result.fetchall()) > 0 + + def getIndexes(self, tbName:str=''): + """ + getIndexes() + ------------ + + This methods gets the list of indexes. + + Parameeter: + - `tbName`: Takes the name of the table. + """ + '''This method is to get the list of the indexes related in the table.''' + if tbName: + query = f"PRAGMA index_list({tbName});" + else: + query = "PRAGMA index_list;" + result = self.execute(query) + return [row[1] for row in result.fetchall()] + + def addIndex(self, table:str, indexName:str, coloumns:str): + '''This method adds an INDEX in the table presented using the provided coloumns.''' + return self.execute(f'CREATE INDEX {indexName} ON {table} ({coloumns});') + + def delIndex(self, tbname:str, idxName:str): + '''This method is to delete an exiting index realted to a table.''' + return self.execute(f'DROP INDEX {idxName} ON {tbname};') + + def createTb(self, tbName: str, columns, primary_key: str = '', addUnique:str='', indexCol:str='', indexName:str=''): + """Creates a table in the database. + + Parameters: + tbName (str): The name of the table to be created. + columns (List[str]): A list of column names and their data types. + primary_key (str, optional): The primary key for the table. Default is 'id'. + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + keys = ['INDEX','SELECT','INSERT','UPDATE','DELETE','FROM','WHERE','JOIN','INNER','LEFT','RIGHT','GROUP BY','ORDER BY','AS','COUNT','SUM','MAX','MIN','AVG','DISTINCT','AND','OR','NOT','BETWEEN','LIKE','IN','NULL','TRUE','FALSE','TOP','LIMIT','OFFSET'] + + for k in keys: + columns = valreplace(columns, k, '_'+k.upper(), 1) + columns = valreplace(columns, k.lower(), '_'+k.lower(), 1) + + + if primary_key != '': + primary_key = primary_key if primary_key not in keys else f'_{primary_key}' + columns = [f"{primary_key} INTEGER PRIMARY KEY AUTOINCREMENT"] + columns + + if addUnique!='': + columns.append(f"UNIQUE({addUnique})") + + query = f"CREATE TABLE IF NOT EXISTS {tbName} ({', '.join(columns)});" + self.execute(query) + + if indexCol!='': + indexName = f'{tbName}_idx' if indexName=='' else indexName + self.addIndex(tbName, indexName, indexCol) + + return True + except Exception as e: + print(f"Error: {e}") + return False + + def renameTb(self, tableName1:str, tableName2:str): + """ This method is to be used to rename the tables provided in the parameter tabeName1 with the value provided in the parameter tableName2.""" + return self.alterTb(tableName1,'RENAME TO', tableName2) + + def cleanTb(self, tableName:str, query:str=''): + """This method is to be used to clean the provided tables clean of any data in it.""" + query = f'WHERE {query}' if query!='' else '' + self.execute(f'DELETE FROM {tableName}{query};') + if query == '': + self.execute(f"DELETE FROM sqlite_sequence WHERE name='{tableName}'") + else: + self.update('sqlite_sequence', {'seq': self.getCount(tableName)}, f"name='{tableName}'") + + def addUniqueColumns(self, table:str, name:str, uniques) -> None: + """ + This method will set unique value to the columns such that if user want to enter a value which already exists then it will reject. + """ + if isinstance(uniques, (list, KeysView, ValuesView)): + uniques = ','.join(uniques) + self.alterTb(table, 'ADD CONSTRAINT', f'{name} UNIQUE({uniques})') + + def delUnique(self, table_name:str, name:str)->None: + """ + This method delete the unique charectersitics of the table column constrain with the provideed name. + """ + self.alterTb(table_name, 'DROP', f'CONSTRAINT {name}') + + def delTb(self, tableName:str): + """This method is for the use of deleting tables if it exists.""" + for i in self.getIndexes(tableName): + self.delIndex(tableName, i) + return self.execute(f'DROP TABLE IF EXISTS {tableName};') + + def addColumn(self, table_name:str, new_column_name, data_type:str, adjacent_column_name:str, column_param:str='', after=True): + """ + Adds a new column to the specified table before or after a specific column. + + Parameters: + table_name (str): The name of the table to add the column to. + new_column_name (list, str, optional): The name of the new column. + data_type (str): The data type for the new column (e.g., "INTEGER", "TEXT", "REAL"). + adjacent_column_name (str): The name of the column before or after which the new column should be added. + after (bool, optional): True to add the new column after the specified column, False to add it before. Default is True. + + Returns: + sqlite3.Cursor: The result of the executed query. + """ + temp_table_name = f"temp_{table_name}" + try: + info = self.get_info(table_name) + columns = info['column_names'] + col_index = [i for i, x in enumerate(self.getColumnNames(table_name)) if adjacent_column_name == x][0] + if after: + col_index+= 1 + if isinstance(new_column_name, list): + for i in new_column_name: + columns.insert(col_index, f"{i} {data_type} {column_param}") + col_index += 1 + elif isinstance(new_column_name, str): + columns.insert(col_index, f"{new_column_name} {data_type} {column_param}") + + self.createTb(temp_table_name, columns) + if info['rows'] > 0 : + data = self.fetch(table_name) + if isinstance(new_column_name, list): + for nm in new_column_name: + data[0][nm]= '' + elif isinstance(new_column_name, str): + data[0][new_column_name]= '' + self.json_insert(temp_table_name, equalizer_dict(data)) + self.delTb(table_name) + self.renameTb(temp_table_name, table_name) + return True + except Exception as e: + self.delTb(temp_table_name) + return False + + def renameColumn(self, table_name:str, old_column_name:str, new_column_name:str): + """ + Renames a column in the specified table. + + Parameters: + table_name (str): The name of the table. + old_column_name (str): The current name of the column to be renamed. + new_column_name (str): The new name for the column. + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + columns = self.getColumnNames(table_name) + + if old_column_name not in columns: + print(f"Error: Column '{old_column_name}' not found in table '{table_name}'.") + return False + + if new_column_name in columns: + print(f"Error: Column '{new_column_name}' already exists in table '{table_name}'.") + return False + + + index = columns.index(old_column_name) + columns[index] = new_column_name + + temp_table_name = f"temp_{table_name}" + self.createTb(temp_table_name, columns) + data = valreplace(self.fetch(table_name), old_column_name, new_column_name, True) + self.json_insert(temp_table_name,) + self.delTb(table_name) + self.renameTb(temp_table_name, table_name) + + return True + except Exception as e: + print(f"Error: {e}") + return False + + def modifyColumn(self, table_name:str, column_name:str, new_column_name:str='', data_type:str='', column_param:str=''): + """ + Modifies a column in the specified table. + + Parameters: + table_name (str): The name of the table. + column_name (str): The name of the column to be modified. + new_column_name (str, optional): The new name for the column. Defaults to ''. + data_type (str, optional): The new data type for the column. Defaults to ''. + column_param (str, optional): Additional column parameters. Defaults to ''. + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + columns_info = self.execute(f"PRAGMA table_info({table_name})").fetchall() + column_names = [info[1] for info in columns_info] + + if column_name not in column_names: + print(f"Error: Column '{column_name}' not found in table '{table_name}'.") + return False + + if new_column_name == '' and data_type == '' and column_param == '': + print("Error: No modifications provided.") + return False + + temp_table_name = f"temp_{table_name}" + original_columns = self.getColumnNames(table_name) + + modified_column = f"{new_column_name} {data_type} {column_param}" if new_column_name else column_name + modified_columns = [modified_column if col == column_name else col for col in original_columns] + + self.createTb(temp_table_name, modified_columns) + self.json_insert(temp_table_name, self.fetch(table_name)) + self.delTb(table_name) + self.renameTb(temp_table_name, table_name) + + print(f"Column '{column_name}' in table '{table_name}' modified successfully.") + return True + except Exception as e: + print(f"Error: {e}") + return False + + def modifyColumns(self, table_name:str, modifications:dict): + """ + Modifies multiple columns in the specified table. + + Parameters: + table_name (str): The name of the table. + modifications (dict): A dictionary where keys are column names and values are dictionaries + containing modification options (new_column_name, data_type, column_param). + + Usage: + ``` + modifications = { + 'column1': {'new_column_name': 'new_column1', 'data_type': 'TEXT'}, + 'column2': {'data_type': 'INTEGER', 'column_param': 'NOT NULL'} + } + db_handler.modifyColumns('your_table_name', modifications) + ``` + + Returns: + bool: True if the operation is successful, False otherwise. + """ + try: + for column_name, options in modifications.items(): + success = self.modifyColumn(table_name, column_name, **options) + if not success: + return False + return True + except Exception as e: + print(f"Error: {e}") + return False + + def removeColumn(self, table_name:str, column_name:str): + """ + Removes a column from the specified table. + + Parameters: + table_name (str): The name of the table to remove the column from. + column_name (str): The name of the column to remove. + + Returns: + sqlite3.Cursor: The result of the executed query. + """ + return self.execute(f"ALTER TABLE {table_name} DROP COLUMN {column_name};") + + def close_connection(self, mesg=None): + """This method for to close all the connections made to the db and aslomend the session.""" + self.db_init.close() + if mesg!=None: + print(mesg) + +class MySqlHandler(): + + def __init__(self, host:str, user:str, password:str, dataBase:str=""): + try: + self.dbConn = myqC.connect(host=host, user=user, passwd=password) + self.cursor = self.dbConn.cursor() + except myqC.Error as err: + if err.errno == myqC.errorcode.ER_ACCESS_DENIED_ERROR: + return 'Error Password!' + else: return err.errno + + + def createTb(self, tableName, columns, Engine:str='InnoDb', tableComment:str=''): + """ + """ + if self.getTables(tableName): raise ValueError(f'This table [{tableName}] already exist!') + column = ','.join(columns) + + query = f"CREATE TABLE `{tableName}` ({column}) ENGINE = \'{Engine}'" + if tableComment != '': + query += f" COMMENT = '{tableComment}'" + + query+= ';' + self.cursor.execute(query) + + def getTables(self, table:str=''): + """ + The method is to get the list of tables form the database. + """ + self.cursor.execute("SHOW TABLES") + tbList = [i[0] for i in self.cursor.fetchall()] + if table!='': return table in tbList + return tbList + + def addIndex(self, tableName:str, index_name:str, columns:list): + """ + This method is to add index to the tables. + """ + + columns = ','.join(columns) + query = f'`{index_name}` ({columns})' + self.alterTb(tableName, 'ADD UNIQUE', query) + + def alterTb(self, tableName:str, catg:str, query:str): + """ + This method is to used to alter tables in the database. + """ + self.cursor.execute(f"ALTER TABLE `{tableName}` {catg} {query};") + + def cleanTb(self, tableName:str): + """ + This method is to clean the table i.e. it will delete all the data from the table. + """ + if self.getTables(tableName): + self.cursor.execute(f'TRUNCATE TABLE `{tableName}`;') + + def delTb(self, tableName:str): + """This method is to delete the specified table.""" + if self.getTables(tableName): + self.cursor.execute(f'DROP TABLE `{tableName}`;') + + def connect_db(self, dataBase:str, create_db:bool=True): + """ + The task of this mathod is to create new databases in the server. + """ + if self.getDbList(dataBase)==False and create_db: + self.createDb(dataBase) + self.dbConn.database = dataBase + + def createDb(self, dataBase:str): + """ + This method is to crreate a new dataBase. + """ + try: + if self.getDbList(dataBase) == True: ValueError('database already exists.') + return self.execute(f'CREATE DATABASE {dataBase}') + except Exception as e: return e + + def getDbList(self, present:str=''): + """ + This method gets the list of databases present in the server. + + """ + self.execute("SHOW DATABASES") + dbList = [i[0] for i in self.cursor.fetchall()] + if present != '': return present in dbList + return dbList + + def execute(self, query:str): + """ + This method is to execute the mysql queries. + """ + try: + return self.cursor.execute(query) + except: + pass + + def delDb(self, dataBase:str): + """ + This method is to delete dataBases. + """ + if self.getDbList(dataBase): + self.execute(f'DROP DATABASE {dataBase}') + else: + raise ValueError('Database does not exist!') + + def close_connection(self): + """ + This method is to close the connection. + """ + self.cursor.close() + self.dbConn.close() \ No newline at end of file diff --git a/FileHandler.py b/FileHandler.py new file mode 100644 index 0000000..604c8c9 --- /dev/null +++ b/FileHandler.py @@ -0,0 +1,288 @@ +import json, csv, os +from openpyxl import Workbook, load_workbook +from PyPDF2 import PdfReader +from DataHandlers import remove_empty_strings, dlist_dict, list_dlist, decode_json + + +def getFiles(file_path:str='', catg:int=0, extention=None, full_path:bool=False): + """ + + Args: + file_path (str): Takes the path of the folder that needs to be looked into. + catg (int): This parameter takes either 1 or 2, where 1 means to only look for files whereas 2 means to lik for directories. Default is 0. Which means both. + + + Returns: + (dict|list): Returns a list of files or directories in the given file. + """ + if file_path=='': + file_path = os.path.dirname(os.path.abspath(__file__)) + + files = os.listdir(file_path) + + if full_path or catg==1 or extention!=None: + files = [os.path.join(file_path, file) for file in files] + + if catg == 1 or extention != None: + if extention!=None: + files = [i for i in files if (getExtention(i)) in extention] + else: + files = [i for i in files if os.path.isfile(i)] + + elif catg == 2: + files = [i for i in files if os.path.isdir(i)] + elif catg == 0: + pass + else: raise ValueError('The parameter passed in catg is invalid. It only accepts 1 or 2 as valid parameter.') + return files if full_path else [get_only_filename(i) for i in files] + +def get_only_filename(full_path): + """_summary_ + + Args: + full_path (str): _description_ + + Returns: + str: _description_ + """ + return os.path.basename(full_path) + +def getFileCatg(file_path:str): + """ + + """ + if os.path.isdir(file_path): + return 2 + elif os.path.isfile(file_path): + return 1 + return None + +def splitFileName(file_path:str)->str: + """ + splitFileName() + --------------- + Returns the name of the file from the filePath given. + + Args: + file_path (str): The locaion of the file relative to the program or the full path of the file. + + Returns: + str: Name of the file. + """ + return os.path.splitext(get_only_filename(file_path))[0] + +def getFileSize(file_path:str): + """ + getFileSize() + ------------- + Gets the size of the file in bytes. + + Args: + file_path (str): _description_ + + Returns: + bytes: _description_ + """ + + if os.path.isfile(file_path): + return os.path.getsize(file_path) + return False + +def getExtention(file:str): + """_summary_ + + Args: + file (str): _description_ + + Returns: + _type_: _description_ + """ + return os.path.splitext(file)[1].replace('.','') + +def get_file_info(self, file:str)->dict: + """ + get_file_info() + --------------- + Gets athe file information if it exists. + + Parameters: + - `file` str: Takes the file name/path. + + Returns: + - dict: Returns the information of the file in the dict format. + - file_name: Name of the file. + - ext: Extention of the file. + - size: Size/ Memory occupied by the file in bytes. + - full_path: The full path of the file. + """ + if fileExists(file)==False: raise FileNotFoundError(f"This file({file}) does not exists") + data:dict = { + "file_name":get_only_filename(file), + "ext": getExtention(file), + "size": getFileSize(file), + "full_path": os.path.dirname(file) + } + return data + +def fileExists(file:str): + """Checks if a file exists or not""" + return True if os.path.exists(file) and os.path.isfile(file) else False + +def write(file_name: str, data, separator='', mode:str='a', write_bytes:bool=False, emptyPervious:bool=False) -> None: + """ + Appends data to a file. + + :param file_name: The name of the file. + :param data: The data to be written to the file. + :param separator: Optional separator to append after the data. + :param mode: Changes the mode to writing any specific format. + :param write_bytes: Changes the mode to write to bytes. + :param emptyPrevious: Cleans the previously inserted data in the file and writes a fresh. + """ + + if emptyPervious: + mode = 'w' + if write_bytes: mode=mode+'b' + + if isinstance(data,(list, tuple, set)): + [write(file_name, d, separator) for d in data] + else: + with open(file_name, mode, encoding='utf-8', errors='ignore') as file: + data_type = type(data) + if data_type == str: + file.write(data) + elif data_type == dict: + json.dump(data, file) + else: TypeError('The data type provided is not supported.') + file.write(separator) + +def read(file_name: str, splitter=None, encode='utf-8', error='ignore', mode:str='r', read_bytes:bool=False, decode_json=False)->str | list | dict | bytes | int | float: + """ + Reads data from a file. + + :param file_name: The name of the file. + :param splitter: Optional splitter to split the data. + :param encode: Encoding of the file. + :param error: How to handle encoding errors. + :param decode_json: Whether to decode JSON-formatted data. + + :return: The read data. + """ + if read_bytes: + mode = 'rb' + encode, error = None, None + + with open(file_name, mode, encoding=encode, errors=error) as file: + data = file.read() + retdata = [] + if splitter is not None: + for ret in remove_empty_strings(data.split(splitter)): + if decode_json: + ret = json.loads(ret) + retdata.append(ret) + return retdata + else: + if decode_json: + return json.loads(data) + else: + return data + +def write_csv(file_name: str, data, header=None): + """ + Writes data to a CSV file. + + :param file_name: The name of the CSV file. + :param data: The data to be written to the CSV file. + :param header: Optional header for the CSV file. + """ + with open(file_name, 'w', newline='', encoding='utf-8', errors='ignore') as csvfile: + csv_writer = csv.writer(csvfile) + if header: + csv_writer.writerow(header) + csv_writer.writerows(data) + +def read_csv(file_name: str, header:bool=False): + """ + read_csv() + ---------- + Reads data from a CSV file. + + Parameter: + - file_name (str): The name of the CSV file. + - header (bool, optional): + + :return: The read data. + """ + with open(file_name, 'r', encoding='utf-8', errors='ignore') as csvfile: + csv_reader = csv.reader(csvfile) + data = [row for row in csv_reader] + if header == True: + return dlist_dict(list_dlist(data[1:], data[0])) + return data + +def write_excel(filename, data, sheetname='Sheet 1'): + workbook = Workbook() + sheet = workbook.active + sheet.title = sheetname + if isinstance(data, list) and isinstance(data[0], dict): + pass + elif isinstance(data, list) and isinstance(data[0], list): + for i, row in enumerate(data): + for j, value in enumerate(row): + sheet.cell(row=i+1, column=j+1, value=value) + elif isinstance(data, dict): + pass + workbook.save(filename) + +def read_excel(file_name: str, sheet_name=None, organize:bool=False): + """ + Reads data from an Excel (XLSX) file. + + :param file_name: The name of the Excel file. + :param sheet_name: Optional sheet name for the Excel file. + + :return: The read data. + """ + workbook = load_workbook(file_name) + sheet_data = {} + if sheet_name!=None and sheet_name in workbook.sheetnames: + sheet = workbook[sheet_name] + sheet_data = [list(row) for row in sheet.iter_rows(values_only=True)] + if organize: + sheet_data = dlist_dict(list_dlist(sheet_data[1:], sheet_data[0])) + else: + for sheet_name in workbook.sheetnames: + sheet = workbook[sheet_name] + data = [list(row) for row in sheet.iter_rows(values_only=True)] + sheet_data[sheet_name] = dlist_dict(list_dlist(data[1:], data[0])) if organize else data + + return sheet_data + +def read_pdf(file_name: str): + """ + Reads text data from a PDF file. + + :param file_name: The name of the PDF file. + + :return: The read text data. + """ + with open(file_name, 'rb') as pdf_file: + pdf_reader = PdfReader(pdf_file) + text = '' + for page in pdf_reader.pages: + text += page.extract_text() + return text + +def read_har(fileName:str): + entries = [] + urls = [] + data = decode_json(read(fileName).replace('\n','').replace('\r', '').replace('\t', ''))['log'] + for i in data['entries'][1:]: + full_url = i['request']['url'] + if full_url not in urls and i['request']['headers'][0]['value']=='www.nseindia.com' and '/api' in full_url and full_url != "https://www.nseindia.com/api/marketStatus": + headers = {hr['name']:hr['value'] for hr in i['request']['headers']} + entries.append({'file':fileName,'full_url':full_url, 'headers':headers, 'response': i['response']['content']}) + urls.append(full_url) + return urls, entries + + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..42f9bc5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright 2024 Coderooz + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..67b6068 --- /dev/null +++ b/Readme.md @@ -0,0 +1,479 @@ +# Custom Functions + +[![Python Version](https://img.shields.io/badge/python-3.12%2B-blue.svg)](https://www.python.org/downloads/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.txt) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg)](CONTRIBUTING.md) +[![Status](https://img.shields.io/badge/status-alpha-orange.svg)](CHANGELOG.md) + +A collection of pre-made Python utilities for everyday tasks. This package provides ready-to-use classes for data handling, database operations, file I/O, and HTTP requests — helping you build projects faster and more effectively. + +## Features + +- **DataHandler**: Timestamp formatting, data normalization, and common data manipulation utilities +- **DbHandler**: Full-featured SQLite database management with CRUD operations, table schema modifications, and index management +- **FileHandler**: Comprehensive file operations including read/write, CSV/Excel/PDF handling, and file system utilities +- **Requester**: Simplified HTTP request handling with common patterns and utilities + +## Installation + +### From PyPI (Recommended) + +```bash +pip install custom_functions +``` + +### From Source + +```bash +# Clone the repository +git clone https://github.com/coderooz/My_simple_functions.git +cd My_simple_functions + +# Install in development mode +pip install -e . + +# Or install with development dependencies +pip install -e ".[dev]" +``` + +### Requirements + +- Python 3.12 or higher +- Dependencies (installed automatically): + - `pytz>=2023.3.0` + - `requests>=2.31.0` + - `db-sqlite3>=0.0.1` + - `pandas>=2.1.3` + - `mysql-connector-python>=2.2.9` + - `openpyxl>=3.1.2` + - `PyPDF2>=3.0.0` + +## Quick Start + +```python +# Import the handlers +from DataHandlers import DataHandler +from DbHandler import DbHandler +from FileHandler import FileHandler +from Requester import Requester + +# Use DataHandler for timestamp formatting +timestamp = DataHandler.timestamp() +print(timestamp) # Output: 2024-01-15 10:30:45 + +# Use DbHandler for database operations +db = DbHandler('my_database.db') +db.createTb('users', ['name TEXT', 'email TEXT']) +db.insert('users', ['John Doe', 'john@example.com']) + +# Use FileHandler for file operations +FileHandler.write('output.txt', 'Hello, World!') +content = FileHandler.read('output.txt') + +# Use Requester for HTTP requests +# requester = Requester() +# response = requester.get('https://api.example.com/data') +``` + +## Detailed Usage + +### DataHandler + +The `DataHandler` class provides utilities for common data manipulation tasks. + +#### `timestamp(given_time=None, format="%Y-%m-%d %H:%M:%S", time_zone=None, normalize='sec')` + +Returns the date and time in the specified format. + +**Parameters:** +- `given_time`: Optional datetime object or timestamp (default: current time) +- `format`: strftime format string (default: `"%Y-%m-%d %H:%M:%S"`) +- `time_zone`: Optional timezone string (e.g., `"Asia/Kolkata"`) +- `normalize`: Normalization level — `'sec'`, `'min'`, or `'hour'` (default: `'sec'`) + +**Returns:** `str` — Formatted timestamp string + +**Example:** +```python +from DataHandlers import DataHandler + +# Current timestamp +print(DataHandler.timestamp()) +# Output: 2024-01-15 10:30:45 + +# Custom format +print(DataHandler.timestamp(format="%Y-%m-%d")) +# Output: 2024-01-15 + +# With timezone +print(DataHandler.timestamp(time_zone="US/Pacific")) +# Output: 2024-01-14 21:00:45 +``` + +--- + +### DbHandler + +The `DbHandler` class provides a comprehensive interface for SQLite database operations. + +#### Initialization + +```python +from DbHandler import DbHandler + +# Creates new database or connects to existing one +db = DbHandler('my_database.db') +``` + +#### Core Methods + +##### `createTb(table_name, columns, insertData=None, addId=False, idKey='id')` + +Creates a new table with optional data insertion. + +**Parameters:** +- `table_name`: Name of the table +- `columns`: List of column definitions (e.g., `['col1 TEXT', 'col2 INT']`) +- `insertData`: Optional data to insert immediately after creation +- `addId`: Whether to add an auto-incrementing primary key (default: `False`) +- `idKey`: Name of the primary key column (default: `'id'`) + +**Example:** +```python +# Create table with columns +db.createTb('users', ['name TEXT', 'email TEXT', 'age INT']) + +# Create with auto ID and initial data +db.createTb('users', ['name TEXT', 'email TEXT'], + insertData=['John', 'john@example.com'], + addId=True) +``` + +##### `insert(table_name, data)` + +Inserts a row into the specified table. + +**Example:** +```python +db.insert('users', ['Alice', 'alice@example.com', 25]) +``` + +##### `fetch(table_name, columns='*', query='')` + +Fetches data from a table with optional filtering. + +**Example:** +```python +# Fetch all rows +all_users = db.fetch('users') + +# Fetch specific columns with condition +result = db.fetch('users', columns='name, email', query="age > 18") +``` + +##### `getCount(table_name, query='')` + +Counts rows in a table, optionally with a filter. + +**Example:** +```python +total = db.getCount('users') +adults = db.getCount('users', query="age >= 18") +``` + +##### `update(table_name, data, query)` + +Updates rows matching the query. + +**Example:** +```python +db.update('users', {'age': 26}, query="name='Alice'") +``` + +##### `execute(query, params=None)` + +Executes raw SQL queries. + +**Example:** +```python +db.execute("DROP TABLE IF EXISTS temp_table") +``` + +#### Additional Methods + +| Method | Description | +|--------|-------------| +| `delTb(table_name)` | Delete a table | +| `renameTb(old_name, new_name)` | Rename a table | +| `getTb()` | List all tables | +| `getTbData(table_name)` | Get all data from a table | +| `alterTb(table_name, operation)` | Alter table structure | +| `getColumnNames(table_name)` | Get column names | +| `get_table_info(table_name)` | Get detailed table info | +| `modifyColumns(table_name, columns)` | Modify column definitions | +| `checkIndex(table_name, index_name)` | Check if index exists | +| `getIndexes(table_name)` | Get all indexes | +| `addIndex(table_name, columns, unique=False)` | Add an index | +| `delIndex(index_name)` | Delete an index | +| `cleanTb(table_name)` | Delete all rows from a table | +| `addColumn(table_name, column_def)` | Add a new column | +| `renameColumn(table_name, old_name, new_name)` | Rename a column | +| `removeColumn(table_name, column_name)` | Remove a column | +| `close_connection()` | Close database connection | + +--- + +### FileHandler + +The `FileHandler` class provides utilities for file system operations and file format handling. + +#### Initialization + +```python +from FileHandler import FileHandler + +# Static class — no initialization needed +# Use directly: FileHandler.method_name() +``` + +#### Core Methods + +##### `getFiles(file_path, catg=0)` + +Gets list of files and/or folders in a directory. + +**Parameters:** +- `file_path`: Path to the directory +- `catg`: Category filter — `0` for both, `1` for files only, `2` for directories only + +**Returns:** `list` of file/directory names + +**Example:** +```python +# Get both files and folders +items = FileHandler.getFiles('./my_folder') + +# Get only files +files = FileHandler.getFiles('./my_folder', catg=1) + +# Get only directories +dirs = FileHandler.getFiles('./my_folder', catg=2) +``` + +##### `get_only_filename(file_path)` + +Extracts just the filename from a path. + +##### `getFileCatg(file_path)` + +Gets the file category/type based on extension. + +##### `splitFileName(file_path)` + +Splits filename into name and extension. + +##### `getExtention(file_path)` + +Gets the file extension. + +##### `read(file_name)` + +Reads and returns file contents. + +##### `write(file_name, data, separator='')` + +Appends data to a file. + +**Parameters:** +- `file_name`: Path to the file +- `data`: Content to write +- `separator`: Optional separator to append after data + +**Example:** +```python +FileHandler.write('output.txt', 'Hello, World!', separator='\n') +``` + +##### `write_over(file_name, data, separator='')` + +Overwrites file contents with new data. + +##### `read_csv(file_name, **kwargs)` + +Reads a CSV file and returns a pandas DataFrame. + +##### `write_csv(file_name, data, **kwargs)` + +Writes data to a CSV file. + +##### `read_excel(file_name, **kwargs)` + +Reads an Excel file and returns a pandas DataFrame. + +##### `write_excel(file_name, data, **kwargs)` + +Writes data to an Excel file. + +##### `read_pdf(file_name)` + +Extracts and returns text content from a PDF file. + +**Example:** +```python +text = FileHandler.read_pdf('document.pdf') +print(text) +``` + +--- + +### Requester + +The `Requester` class provides simplified HTTP request handling. + +```python +from Requester import Requester + +# Initialize +requester = Requester() + +# GET request +response = requester.get('https://api.example.com/data') + +# POST request +response = requester.post('https://api.example.com/data', json={'key': 'value'}) + +# With custom headers +response = requester.get('https://api.example.com/data', + headers={'Authorization': 'Bearer token'}) +``` + +## Project Structure + +``` +custom_functions/ +├── DataHandlers.py # Data manipulation utilities +├── DbHandler.py # SQLite database operations +├── FileHandler.py # File I/O and format handling +├── Requester.py # HTTP request utilities +├── __init__.py # Package initialization +├── setup.py # Package distribution setup +├── pyproject.toml # Modern Python project configuration +├── LICENSE.txt # MIT License +├── README.md # This file +├── CHANGELOG.md # Version history +├── CONTRIBUTING.md # Contribution guidelines +├── CODE_OF_CONDUCT.md # Community guidelines +├── SECURITY.md # Security policy +├── .gitignore # Git ignore rules +├── .editorconfig # Editor configuration +└── .github/ # GitHub-specific files + ├── ISSUE_TEMPLATE/ # Issue templates + ├── workflows/ # CI/CD workflows + ├── CODEOWNERS # Code ownership + ├── dependabot.yml # Dependency updates + ├── labels.yml # Issue labels + ├── PULL_REQUEST_TEMPLATE.md + └── FUNDING.yml # Sponsorship information +``` + +## Development + +### Setup Development Environment + +```bash +# Clone and set up virtual environment +git clone https://github.com/coderooz/My_simple_functions.git +cd My_simple_functions +python -m venv venv +source venv/bin/activate # Windows: venv\Scripts\activate + +# Install with dev dependencies +pip install -e ".[dev]" +``` + +### Running Tests + +```bash +# Run all tests +pytest + +# Run with coverage +pytest --cov=custom_functions --cov-report=html + +# Run specific test file +pytest tests/test_datahandler.py -v +``` + +### Code Quality + +```bash +# Format code +black . + +# Sort imports +isort . + +# Lint +flake8 . + +# Run all checks +black . && isort . && flake8 . +``` + +## Contributing + +We welcome contributions of all kinds! Please read our [Contributing Guide](CONTRIBUTING.md) for details on: + +- How to set up your development environment +- Our coding standards and conventions +- How to submit pull requests +- How to report bugs or request features + +### Quick Start for Contributors + +1. Fork the repository +2. Create a feature branch (`git checkout -b feature/amazing-feature`) +3. Make your changes +4. Run tests and linting +5. Commit your changes (`git commit -m 'feat: add amazing feature'`) +6. Push to the branch (`git push origin feature/amazing-feature`) +7. Open a Pull Request + +Please read our [Code of Conduct](CODE_OF_CONDUCT.md) before contributing. + +## Versioning + +We use [Semantic Versioning](https://semver.org/) (SemVer) for versioning. See the [CHANGELOG.md](CHANGELOG.md) for a list of available versions and the changes in each release. + +## License + +This project is licensed under the MIT License — see the [LICENSE.txt](LICENSE.txt) file for details. + +## Author + +**Ranit Saha** (Coderooz) + +- GitHub: [@Coderooz](https://github.com/coderooz) +- Website: [https://coderooz.in](https://coderooz.in) +- Email: [contact@coderooz.in](mailto:contact@coderooz.in) +- Contact: [https://coderooz.in/contact](https://coderooz.in/contact?subject=[message_subject]&message=[message]) + +## Acknowledgments + +- Thanks to all contributors who help improve this project +- Built with the goal of reducing repetitive coding tasks + +## Support + +If you find this project helpful, consider supporting: + +- [Sponsor on GitHub](https://github.com/sponsors/Coderooz) +- [Contact for custom sponsorship](https://coderooz.in/contact?subject=Sponsorship) +- [Star this repository](https://github.com/coderooz/My_simple_functions) ⭐ + +--- + +

+ Made with ❤️ by CodeRooz +

diff --git a/Requester.py b/Requester.py new file mode 100644 index 0000000..c4e2604 --- /dev/null +++ b/Requester.py @@ -0,0 +1,440 @@ +import requests, random, time, websockets, aiohttp, asyncio +from urllib.parse import urlparse, parse_qsl, urlencode +from FileHandler import read + +class Requester: + """ + Requester() + =========== + + Requester is a class for making HTTP & HTTPS requests easier speciall dureing the time of development. + + """ + + def __init__(self, agent:list=[], header:dict={}, proxy:list=[], ref:list=[], ref_file:str='', proxy_file:str='', agent_file:str='', set_agent:bool=True, set_header:bool=True, set_ref:bool=False, set_proxy:bool=False, break_pt:list=[]): + self.agent, self.ref, self.proxy, self.header = 0, '', 0, 0 + self.break_pt = break_pt + self.ws = None + + self.start_up() + + if agent_file != '': + self.agent = read(agent_file, '\n') + elif agent != []: + self.agent = agent + else: + self.agent = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'] + + if ref_file != '': + self.ref = read(ref_file,'\n') + elif ref is not None: + self.ref = ref + + if proxy_file != '': + self.proxy = read(proxy_file,'\n') + elif proxy != []: + self.proxy = proxy + else: + self.proxy= [] + + if set_header: + self.header = header + + # if self._check_connection()==False: + # raise ConnectionRefusedError('There is some issues with the internet Connection. Please the internet connection before performing any requests.') + + def get_proxy(self): + """ + This method gives a proxy url randomly. + """ + if self.proxy != []: return random.choice(self.proxy) + + def headers(self, agent:str='', ref:str='', header:dict={}, change:bool=False): + """ + headers() + -------- + The header method of the class gives out the headers necessay for the requests + Args: + agent (str, optional): _description_. Defaults to None. + ref (str, optional): _description_. Defaults to None. + header (dict, optional): _description_. Defaults to None. + change (bool, optional): This parameter takes in True or False which determines whether the headers will change or remain the same based on the data passed respectively. Defaults to False. + + Returns: + _type_: _description_ + """ + headers = {'connection': 'keep-alive','accept-Encoding': 'gzip, deflate, br','cache-Control': 'max-age=0','dnt': '1','upgrade-insecure-requests': '1','user-agent': '','accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9','sec-fetch-site': 'same-origin','sec-fetch-mode': 'navigate','sec-fetch-user': '?1', 'referer': '','accept-language': 'en-GB,en-US;q=0.9,en;q=0.8'} + + if (self.header!=0) and (header=={}) and (change==False): + header = self.header + elif (header!={}): + headers = {**headers, **{k:v for k,v in header.items()}} + if change: + self.header = headers + + if agent=='' and isinstance(self.agent, list): + agent = random.choice(self.agent) + elif agent!='': + agent = agent + + if ref=='' and self.ref!=[]: + if isinstance(self.ref, list): + ref=random.choice(self.ref) + elif isinstance(self.ref, str): + ref = self.ref + + headers['user-agent'] = str(agent) + headers['referer'] = str(ref) + + return headers + + def set_url_params(self, url, params:dict={}): + """Set the parameters in a URL. + + Parameters: + url (str): The base URL. + params (dict): A dictionary of parameters to set in the URL. + + Returns: + str: The URL with the parameters set. + """ + if params!={}: + params = {k: v for k, v in params.items() if v!=None or v!=''} + encoded_params = urlencode(params) + url = f"{url}?{encoded_params}" + return url + + def get_urlinfo(self, url): + """ + This method returns the info of the url. + """ + ino = urlparse(url) + return {'scheme':ino.scheme, 'hostname':ino.hostname, 'path':ino.path,'params':dict(parse_qsl(ino.params)), 'query':ino.query, 'fragment':ino.fragment} + + def parse_url_parameters(self, url:str): + """ + This method gets parameters from the url. + """ + return self.get_urlinfo(url)['params'] + + def request(self, url, method='get', params=None, data=None, json:dict={}, header:dict={}, cookies=None, timeout=5, redirect=True, verify=True, proxy=None, ref:str='', agent:str='', break_pt:list=[], setHeader:bool=False): + + break_pt = self.break_pt if break_pt is [] else break_pt + if break_pt != []: time.sleep(random.uniform(break_pt[0], break_pt[1])) + proxy = self.get_proxy() if proxy is None else proxy + header = None if header=={} else self.headers(agent, ref, header, setHeader) + + if method.lower() == 'get': + ret = requests.get(url, params=params, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + elif method.lower() == 'post': + ret = requests.post(url, params=params, data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + elif method.lower() == 'put': + ret = requests.put(url, params=params, data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + elif method.lower() == 'patch': + ret = requests.patch(url, params=params, data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + elif method.lower() == 'delete': + ret = requests.delete(url,params=params,data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore + return ret #type:ignore + + def requestSessions(self, url:str, method:str='get', params=None, data=None, json=None, header=None, cookies=None, timeout:int=5, sessions=None, redirect=True, verify=True, proxy=None, ref:str='', agent:str='', pre_request:bool=False, break_pt:list=[]): + """ + This method (requestSessions) is do make request based on the sessions. + + Args: + url (str): Take the url that is to be requested. + method (str, optional): Takes the method in which the url is requested. Defaults to 'get'. + params (dict, optional): Takes the parameters of the url in a dict format. Defaults to None. + data (_type_, optional): _description_. Defaults to None. + json (_type_, optional): _description_. Defaults to None. + header (_type_, optional): _description_. Defaults to None. + cookies (_type_, optional): _description_. Defaults to None. + timeout (int, optional): This takes the time in seconds how much time will the systems wait for the respose . Defaults is 5 (5 sec). + sessions (_type_, optional): Takes the sessions data. Defaults to None. + redirect (bool, optional): _description_. Defaults to True. + verify (bool, optional): _description_. Defaults to True. + proxy (_type_, optional): _description_. Defaults to None. + ref (_type_, optional): Takes the reffered url or the url that will display where its requested froms. Defaults to None. + agent (_type_, optional): Takes the user-agent detials. Defaults to None. + pre_request (bool, optional): This parameter is responsible for adding sessions to the requsted url if given `True`. Defaults to False. + break_pt (_type_, optional): _description_. Defaults to None. + + Returns: + resonse: Returns the response of the requested url. + sessions: Returns the sessions created in the requseting process. + """ + break_pt = self.break_pt if break_pt == [] else break_pt + if break_pt != []: time.sleep(random.uniform(break_pt[0], break_pt[1])) + proxy = self.get_proxy() if proxy is None else proxy + header = self.headers(agent, ref) if header is None else header + + s = sessions if sessions!=None else requests.sessions.Session() + if header != None: s.headers.update(header) + if cookies!=None: s.cookies.update(cookies) + if proxy != None: s.proxies.update(proxy) + s.verify = verify + s.timeout = timeout + s.allow_redirects = redirect + + if pre_request and sessions==None: + if isinstance(pre_request, bool): + info = self.get_urlinfo(url) + prevon = info['scheme'] + '://'+ info['hostname'] + s.get(prevon) + else: s.get(pre_request) + + if method.lower() == 'get': + ret = s.get(url, params=params) + elif method.lower() == 'post': + ret = s.post(url, params=params, data=data, json=json) + elif method.lower() == 'put': + ret = s.put(url, params=params, data=data, json=json) + elif method.lower() == 'patch': + ret = s.patch(url, params=params, data=data, json=json) + elif method.lower() == 'delete': + ret = s.delete(url,params=params,data=data, json=json) + return ret, s + + def start_up(self): + """ + + """ + tables = [ + {"table_name":"requestList", "column_names": ["id INTEGER PRIMARY KEY AUTOINCREMENT", "url TEXT", "data_size REAL", "referer TEXT"], 'indexes':'', 'unique':''}, + ] + for i in tables: + pass + + def connect_websocket(self, url, on_message=None, on_error=None, on_close=None): + """Connect to a WebSocket server at the specified URL. + + Parameters: + url (str): The URL of the WebSocket server. + on_message (function): A function to be called when a message is received. + on_error (function): A function to be called when an error occurs. + on_close (function): A function to be called when the connection is closed. + + Returns: + websocket.WebSocket: The WebSocket connection. + """ + self.ws = websockets.WebSocketApp(url, on_message=on_message, on_error=on_error, on_close=on_close) + self.ws.run_forever() + return self.ws + + def send_websocket_message(self, message): + """Send a message over a WebSocket connection. + + Parameters: + ws (websocket.WebSocket): The WebSocket connection. + message (str): The message to send. + """ + self.ws.send(message) + + def close_websocket(self): + """Close a WebSocket connection. + + Parameters: + ws (websocket.WebSocket): The WebSocket connection. + """ + self.ws.close() + + def check_connection(self, url:str=''): + """ + check_connections() + ------------------- + + This method is to check if there is internet connection avaiable or not. + + Returns: + -------- + returns: This metho returns a bool (True or False). Returns `True` if conncetion is available and `False` if not available. + """ + try: + url = url if url!='' else 'https://www.google.com' + res = self.request(url) + res.raise_for_status() + return True + except requests.RequestException: + return False + +class AsyncRequester: + """ + AsyncRequester() + ================ + + AsyncRequester is a class for making HTTP & HTTPS requests easier especially during the time of development. + """ + + def __init__(self, agent=[], header={}, proxy=[], ref=[], ref_file='', proxy_file='', agent_file='', set_agent=True, set_header=True, set_ref=False, set_proxy=False, break_pt=[]): + self.agent, self.ref, self.proxy, self.header = 0, '', 0, 0 + self.break_pt = break_pt + + if agent_file != '': + self.agent = read(agent_file, '\n') + elif agent != []: + self.agent = agent + else: + self.agent = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'] + + if ref_file != '': + self.ref = read(ref_file, '\n') + elif ref is not None: + self.ref = ref + + if proxy_file != '': + self.proxy = read(proxy_file, '\n') + elif proxy != []: + self.proxy = proxy + else: + self.proxy = [] + + if set_header: + self.header = header + + async def get_proxy(self): + """ + This method gives a proxy url randomly. + """ + if self.proxy != []: + return random.choice(self.proxy) + + async def headers(self, agent='', ref='', header=None, change=False): + """ + The header method of the class gives out the headers necessary for the requests. + + Args: + agent (str, optional): User-agent string. Defaults to ''. + ref (str, optional): Referer URL. Defaults to ''. + header (dict, optional): Additional headers. Defaults to None. + change (bool, optional): Whether to change headers or not. Defaults to False. + + Returns: + dict: Request headers. + """ + headers = {'connection': 'keep-alive', 'accept-Encoding': 'gzip, deflate, br', 'cache-Control': 'max-age=0', 'dnt': '1', 'upgrade-insecure-requests': '1', 'user-agent': '', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'sec-fetch-site': 'same-origin', 'sec-fetch-mode': 'navigate', 'sec-fetch-user': '?1', 'referer': '', 'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8'} + + if (self.header != {}) and (header is None) and (not change): + return self.header + elif (header is not None): + self.header = header + return header + else: + if (agent == '') and (type(self.agent) is list): + agent = random.choice(self.agent) + elif agent != '': + agent = agent + + if ref == '' and self.ref != []: + if isinstance(self.ref, list): + ref = random.choice(self.ref) + elif isinstance(self.ref, str): + ref = self.ref + + headers['user-agent'] = str(agent) + headers['referer'] = str(ref) + + return headers + + async def set_url_params(self, url, params={}): + """Set the parameters in a URL.""" + if params != {}: + params = {k: v for k, v in params.items() if v != None or v != ''} + encoded_params = urlencode(params) + url = f"{url}?{encoded_params}" + return url + + async def get_urlinfo(self, url): + """Return the info of the URL.""" + ino = urlparse(url) + return {'scheme':ino.scheme, 'hostname':ino.hostname, 'path':ino.path,'params':dict(parse_qsl(ino.params)), 'query':ino.query, 'fragment':ino.fragment} + + async def parse_url_parameters(self, url): + """Get parameters from the URL.""" + return await self.get_urlinfo(url)['params'] + + async def request(self, url, method='get', params=None, data=None, json=None, header=None, cookies=None, timeout=5, redirect=True, verify=True, proxy=None, ref='', agent='', break_pt=[]): + """Make an asynchronous HTTP request.""" + break_pt = self.break_pt if break_pt == [] else break_pt + if break_pt != []: + await asyncio.sleep(random.uniform(break_pt[0], break_pt[1])) + proxy = await self.get_proxy() if proxy is None else proxy + header = await self.headers(agent, ref) if header is None else header + + async with aiohttp.ClientSession() as session: + async with session.request(method, url, params=params, data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify_ssl=verify, proxy=proxy) as response: + return await response.text(), response + + async def requestSessions(self, url, method='get', params=None, data=None, json=None, header=None, cookies=None, timeout=5, sessions=None, redirect=True, verify=True, proxy=None, ref='', agent='', pre_request=False, break_pt=[]): + """Make an asynchronous HTTP request with sessions.""" + break_pt = self.break_pt if break_pt == [] else break_pt + if break_pt != []: + await asyncio.sleep(random.uniform(break_pt[0], break_pt[1])) + proxy = await self.get_proxy() if proxy is None else proxy + header = await self.headers(agent, ref) if header is None else header + + s = sessions if sessions is not None else aiohttp.ClientSession() + if header is not None: + s.headers.update(header) + if cookies is not None: + s.cookies.update(cookies) + s.connector.verify_ssl = verify + s.connector.timeout = aiohttp.ClientTimeout(total=timeout) + s.connector.allow_redirects = redirect + + if pre_request and sessions is None: + if isinstance(pre_request, bool): + info = await self.get_urlinfo(url) + prevon = info['scheme'] + '://' + info['hostname'] + await s.get(prevon) + else: + await s.get(pre_request) + + async with s.request(method, url, params=params, data=data, json=json) as response: + return await response.text(), response + + async def connect_websocket(self, url, on_message=None, on_error=None, on_close=None): + """Connect to a WebSocket server at the specified URL.""" + async with websockets.connect(url, on_message=on_message, on_error=on_error, on_close=on_close) as ws: + await ws.wait_closed() + return ws + + async def send_websocket_message(self, ws, message): + """Send a message over a WebSocket connection.""" + await ws.send(message) + + async def close_websocket(self, ws): + """Close a WebSocket connection.""" + await ws.close() + + async def check_connection(self, url=''): + """ + check_connections() + ------------------- + + This method is to check if there is an internet connection available or not. + + Args: + url (str, optional): URL to check the internet connection. Defaults to ''. + + Returns: + bool: True if connection is available, False otherwise. + """ + try: + url = url if url != '' else 'https://www.google.com' + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + response.raise_for_status() + return True + except aiohttp.ClientError: + return False + + + +if __name__ == '__main__': + requester = Requester() + # Make a synchronous HTTP GET request + response_text, response = requester.request('https://api.example.com/data') + # Print the response text + print(response_text) + + + diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..4496fee --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,63 @@ +# Security Policy + +## Supported Versions + +The following versions of the project are currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 0.0.x | :white_check_mark: | + +## Reporting a Vulnerability + +We take the security of this project seriously. If you discover a security vulnerability, please follow the responsible disclosure process below. + +### How to Report + +**DO NOT** open a public GitHub issue for security vulnerabilities. + +Instead, please report security vulnerabilities through one of these methods: + +1. **Email**: [contact@coderooz.in](mailto:contact@coderooz.in) +2. **Contact Form**: [CodeRooz Contact](https://coderooz.in/contact?subject=[Security_Vulnerability_Report]) + +Please include the following information in your report: + +- Description of the vulnerability +- Steps to reproduce the issue +- Potential impact of the vulnerability +- Any suggested fixes (if you have them) + +### What to Expect + +- **Acknowledgment**: You will receive an acknowledgment of your report within **48 hours** +- **Initial Assessment**: We will provide an initial assessment within **7 days** +- **Updates**: We will keep you informed of our progress throughout the process +- **Resolution**: We aim to resolve critical vulnerabilities within **30 days** + +### Security Best Practices for Users + +When using this package: + +1. **Keep Updated**: Always use the latest version of the package +2. **Review Dependencies**: Regularly review and update your project dependencies +3. **Input Validation**: Always validate and sanitize user inputs before passing them to any functions +4. **Secure Credentials**: Never hardcode sensitive information like API keys, passwords, or database credentials +5. **Database Security**: When using `DbHandler`, ensure your database files have appropriate file permissions + +### Security Measures in This Project + +- Input validation on all public methods +- Parameterized SQL queries to prevent SQL injection +- Secure file handling practices +- Regular dependency updates via Dependabot +- Automated security scanning in CI/CD pipelines + +### Bug Bounty + +Currently, this project does not offer a bug bounty program. However, we greatly appreciate responsible disclosure and will credit reporters in our security advisories (unless they prefer to remain anonymous). + +--- + +**Author**: Ranit Saha +**Website**: [https://coderooz.in](https://coderooz.in) diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..f0d6a13 --- /dev/null +++ b/__init__.py @@ -0,0 +1 @@ +all = ['DataHandler', 'Dbhandler', 'FileHandler', 'Requester'] \ No newline at end of file diff --git a/custom-functions.project-mcp.json b/custom-functions.project-mcp.json new file mode 100644 index 0000000..74899c9 --- /dev/null +++ b/custom-functions.project-mcp.json @@ -0,0 +1,144 @@ +{ + "project": { + "name": "custom-functions", + "scope": "project", + "environment": "development" + }, + "connection": { + "strategy": "runtime-first", + "preferredPortRange": [ + 47000, + 47020 + ], + "fallbackPorts": [ + 47000, + 47001, + 47005, + 47010 + ], + "retry": { + "maxRetries": 5, + "backoff": "exponential", + "baseDelay": 200, + "maxDelay": 3200 + }, + "timeout": 30000, + "maxConcurrentRequests": 10, + "healthCheck": { + "enabled": true, + "timeout": 5000, + "retries": 3, + "interval": 30000 + } + }, + "agent": { + "defaultAgent": "mcp-orchestrator", + "autoRegister": true, + "heartbeatInterval": 30000, + "permissions": { + "allowToolExecution": true + }, + "behavior": { + "autoOptimize": true, + "allowSelfModification": false, + "maxRetainedContexts": 100, + "autoMemoryCleanup": true + } + }, + "behavior": { + "ignore": [ + "node_modules/", + ".git/", + "logs/", + "dist/", + "build/", + ".next/", + "coverage/", + "*.log" + ], + "askBefore": [ + "destructive_actions" + ], + "autoApprove": [ + "read_operations" + ] + }, + "features": { + "multiAgent": true, + "chat": true, + "messaging": true, + "feedback": true, + "emulator": true, + "memory": { + "enabled": true, + "optimization": true, + "versioning": true + }, + "tasks": { + "concurrency": "atomic", + "retry": true, + "maxRetries": 3 + } + }, + "security": { + "inputSanitization": true, + "agentIsolation": true, + "idempotency": true + }, + "logging": { + "level": "minimal", + "errorsOnly": true, + "enabled": true + }, + "rules": { + "ignore": [ + "node_modules/", + ".git/", + "dist/", + "build/", + ".next/", + "coverage/", + "*.log" + ], + "protected": [ + "core/", + "infrastructure/" + ], + "scanExtensions": [ + ".js", + ".ts", + ".jsx", + ".tsx", + ".json", + ".md", + ".yml", + ".yaml" + ] + }, + "execution": { + "parallelAgents": true, + "maxAgents": 10, + "agentTimeout": 60000 + }, + "setup": { + "autoConfigure": true, + "autoHeal": true, + "maxSetupAttempts": 3, + "requiredEnvVars": [ + "MCP_SCOPE", + "MCP_PROJECT" + ], + "optionalEnvVars": [ + "MCP_AGENT", + "NODE_ENV", + "LOG_LEVEL" + ] + }, + "plugins": { + "emulator": { + "enabled": true, + "autoDetect": true + } + }, + "policies": {} +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..079a409 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,152 @@ +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "custom_functions" +version = "0.0.1" +description = "Pre-made Python functions for everyday tasks" +readme = "README.md" +license = {text = "MIT"} +authors = [ + {name = "Ranit Saha", email = "contact@coderooz.in"}, +] +maintainers = [ + {name = "Ranit Saha", email = "contact@coderooz.in"}, +] +keywords = [ + "python", + "file-handling", + "requests", + "sqlite", + "database", + "data-handling", + "utilities", + "helper-functions", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Utilities", +] +requires-python = ">=3.12" +dependencies = [ + "pytz>=2023.3.0", + "requests>=2.31.0", + "db-sqlite3>=0.0.1", + "pandas>=2.1.3", + "mysql-connector-python>=2.2.9", + "openpyxl>=3.1.2", + "PyPDF2>=3.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-cov>=4.0", + "twine>=4.0.2", + "black>=23.0", + "isort>=5.0", + "flake8>=6.0", + "mypy>=1.0", +] + +[project.urls] +Homepage = "https://github.com/coderooz/My_simple_functions" +Repository = "https://github.com/coderooz/My_simple_functions" +"Bug Tracker" = "https://github.com/coderooz/My_simple_functions/issues" +Changelog = "https://github.com/coderooz/My_simple_functions/blob/main/CHANGELOG.md" +Documentation = "https://github.com/coderooz/My_simple_functions#readme" +"Author Website" = "https://coderooz.in" + +[tool.setuptools] +packages = ["custom_functions"] + +[tool.setuptools.package-dir] +"" = "." + +[tool.black] +line-length = 120 +target-version = ['py312'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist + | __pycache__ +)/ +''' + +[tool.isort] +profile = "black" +line_length = 120 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true + +[tool.flake8] +max-line-length = 120 +max-complexity = 10 +exclude = [ + ".git", + "__pycache__", + "build", + "dist", + "*.egg-info", + ".eggs", + ".tox", + ".venv", + "venv", +] +per-file-ignores = [ + "__init__.py:F401,F403", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = "-v --tb=short" + +[tool.mypy] +python_version = "3.12" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true + +[tool.coverage.run] +source = ["custom_functions"] +omit = [ + "tests/*", + "setup.py", + "__init__.py", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "pass", + "raise ImportError", +] +show_missing = true +fail_under = 0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7a403a3 --- /dev/null +++ b/setup.py @@ -0,0 +1,31 @@ +from setuptools import setup, find_packages +import codecs +import os + +here = os.path.abspath(os.path.dirname(__file__)) + +with codecs.open(os.path.join(here, "README.md"), encoding="utf-8") as fh: + long_description = "\n" + fh.read() + +VERSION = '0.0.1' +DESCRIPTION = 'Pre-made Python functions.' +LONG_DESCRIPTION = 'A package that allows to build other projects faster and more effectively. This package is made for preventing repetatively writing simple codes and to reduce unnecessary the possibility of a bug appearing in the code while reducing the production time required.' + +# Setting up +setup( + name="custom_functions", + version=VERSION, + author="Coderooz (Ranit Saha)", + author_email="", + description=DESCRIPTION, + long_description_content_type="text/markdown", + long_description=long_description, + packages=find_packages(), + znstall_requires=['pytz>=2023.3.0','requests>=2.31.0', 'db-sqlite3>=0.0.1', 'pandas>=2.1.3',' mysql-connector-python>=2.2.9', 'openpyxl>=3.1.2','PyPDF2>=3.0.0'], + keywords=['python', 'file', 'fileHandleing', 'requests', 'requestsHandling', 'sqlite', 'sqlDbHandling', 'mysql'], + classifiers=["License :: OSI Approved :: MIT Licence","Development Status :: 1 - Planning","Intended Audience :: Developers", + "Programming Language :: Python :: 3","Operating System :: Unix","Operating System :: MacOS :: MacOS X","Operating System :: Microsoft :: Windows"], + licence = "MIT", + extras_require={"dev": ["pytest>=7.0", "twine>=4.0.2"]}, + python_requires=">=3.12.0" +) \ No newline at end of file From 0d5b506180b18ad879ef2d1272d0d17efe852b61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 May 2026 01:53:52 +0000 Subject: [PATCH 6/6] deps(deps): update mysql-connector-python requirement Updates the requirements on [mysql-connector-python](https://github.com/mysql/mysql-connector-python) to permit the latest version. - [Changelog](https://github.com/mysql/mysql-connector-python/blob/trunk/CHANGES.txt) - [Commits](https://github.com/mysql/mysql-connector-python/compare/8.0.4...9.7.0) --- updated-dependencies: - dependency-name: mysql-connector-python dependency-version: 9.7.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 079a409..d9d89a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "requests>=2.31.0", "db-sqlite3>=0.0.1", "pandas>=2.1.3", - "mysql-connector-python>=2.2.9", + "mysql-connector-python>=9.7.0", "openpyxl>=3.1.2", "PyPDF2>=3.0.0", ]