diff --git a/.editorconfig b/.editorconfig index db64f3d..0fd43cd 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,36 +1,43 @@ -# EditorConfig helps maintain consistent coding styles +# Editor configuration file # https://editorconfig.org root = true [*] -indent_style = space -indent_size = 4 -end_of_line = lf charset = utf-8 -trim_trailing_whitespace = true +end_of_line = lf +indent_size = 4 +indent_style = space insert_final_newline = true +trim_trailing_whitespace = true -[*.{yml,yaml}] +[*.md] indent_size = 2 +trim_trailing_whitespace = false -[*.{json,jsonc}] +[*.yml] indent_size = 2 -[*.md] -trim_trailing_whitespace = false +[*.yaml] +indent_size = 2 -[*.{py,pyw}] -indent_size = 4 +[*.json] +indent_size = 2 -[*.{js,jsx,ts,tsx}] +[*.toml] indent_size = 2 -[*.{html,htm,css}] +[*.cfg] indent_size = 2 -[*.sh] +[*.ini] indent_size = 2 [Makefile] indent_style = tab + +[*.{bat,cmd,ps1}] +end_of_line = crlf + +[*.ps1] +indent_size = 2 diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 6b3ed38..0000000 --- a/.gitattributes +++ /dev/null @@ -1,82 +0,0 @@ -# Auto detect text files and perform LF normalization -* text=auto - -# Python -*.py text diff=python -*.pyw text diff=python -*.pyx text diff=python -*.pxd text diff=python - -# C/C++ -*.c text diff=cpp -*.cpp text diff=cpp -*.h text diff=cpp -*.hpp text diff=cpp - -# Shell scripts -*.sh text eol=lf -*.bash text eol=lf - -# PowerShell -*.ps1 text eol=crlf -*.psm1 text eol=crlf -*.psd1 text eol=crlf - -# Batch -*.bat text eol=crlf -*.cmd text eol=crlf - -# YAML -*.yml text -*.yaml text - -# JSON -*.json text - -# Markdown -*.md text diff=markdown -*.mdx text diff=markdown - -# HTML -*.html text diff=html -*.htm text diff=html - -# CSS -*.css text diff=css - -# JavaScript -*.js text diff=javascript -*.jsx text diff=javascript -*.ts text diff=javascript -*.tsx text diff=javascript - -# TOML -*.toml text - -# Config files -*.cfg text -*.ini text -*.conf text - -# Docker -Dockerfile text - -# Git -.gitignore text -.gitattributes text - -# Binary files -*.png binary -*.jpg binary -*.jpeg binary -*.gif binary -*.ico binary -*.svg text -*.webp binary -*.mp3 binary -*.mp4 binary -*.pdf binary -*.zip binary -*.tar.gz binary -*.7z binary -*.rar binary diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bc5fc70..73f45ab 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,15 +1,21 @@ -# Code Owners for My_simple_functions -# These owners will be automatically requested for review -# when someone opens a pull request. +# These are the users/teams that will be automatically added as reviewers to PRs +# that affect certain parts of the codebase. -# Default owner for everything -* @coderooz +# Owner gets notified for all changes +* @Coderooz -# Python code -*.py @coderooz +# Python files +*.py @Coderooz + +# Configuration files +*.yml @Coderooz +*.yaml @Coderooz +*.toml @Coderooz +*.cfg @Coderooz +*.ini @Coderooz # Documentation -*.md @coderooz +*.md @Coderooz -# GitHub configuration -.github/ @coderooz +# GitHub workflows +.github/ @Coderooz diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..1a1e62f --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,13 @@ +# These are supported funding model platforms + +github: Coderooz +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +custom: ['https://coderooz.in/contact?subject=Sponsorship'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 151f9b0..f568189 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,37 +1,42 @@ --- -name: Bug Report -about: Report a bug to help us improve -title: '' -labels: bug +name: πŸ› Bug Report +about: Create a report to help us improve +title: '[BUG]: ' +labels: ['bug', 'needs-triage'] assignees: '' --- -## Describe the Bug - +## Bug Description A clear and concise description of what the bug is. -## To Reproduce - +## Steps To Reproduce Steps to reproduce the behavior: - -1. -2. -3. +1. Import '...' +2. Call function '....' +3. See error ## Expected Behavior - A clear and concise description of what you expected to happen. -## Screenshots +## Actual Behavior +What actually happened, including any error messages or unexpected output. -If applicable, add screenshots to help explain your problem. +## Code Sample +```python +# Minimal code snippet that reproduces the issue +``` ## Environment - -- OS: [e.g., Windows 11, Ubuntu 22.04] -- Python Version: [e.g., 3.12.0] -- Package Version: [e.g., 1.0.0] +- **OS**: [e.g., Windows 10, Ubuntu 22.04, macOS 13] +- **Python Version**: [e.g., 3.12.0] +- **Package Version**: [e.g., 0.0.1] +- **Installation Method**: [e.g., pip, from source] ## Additional Context +Add any other context about the problem here, such as: +- Screenshots +- Log output +- Related issues -Add any other context about the problem here. +## Possible Solution +If you have ideas on how to fix this, please share them here. diff --git a/.github/ISSUE_TEMPLATE/docs_improvement.md b/.github/ISSUE_TEMPLATE/docs_improvement.md new file mode 100644 index 0000000..78ee5ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/docs_improvement.md @@ -0,0 +1,27 @@ +--- +name: πŸ“ Documentation Improvement +about: Suggest improvements to documentation +title: '[DOCS]: ' +labels: ['documentation', 'needs-triage'] +assignees: '' +--- + +## Documentation Issue +Describe what's missing, unclear, or incorrect in the current documentation. + +## Location +Where is the documentation issue located? +- [ ] README.md +- [ ] Function docstrings +- [ ] Code comments +- [ ] CONTRIBUTING.md +- [ ] Other (please specify) + +## Suggested Improvement +A clear and concise description of what should be added or changed. + +## Why This Matters +Explain why this documentation improvement would be valuable to users or contributors. + +## Additional Context +Add any other context or screenshots about the documentation issue here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index e4da9c1..be604f3 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,24 +1,32 @@ --- -name: Feature Request +name: ✨ Feature Request about: Suggest an idea for this project -title: '' -labels: enhancement +title: '[FEATURE]: ' +labels: ['enhancement', 'needs-triage'] assignees: '' --- -## Is your feature request related to a problem? - -A clear and concise description of what the problem is. -Ex. I'm always frustrated when [...] - -## Describe the Solution You'd Like +## Problem Statement +Is your feature request related to a problem? Please describe. +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +## Proposed Solution A clear and concise description of what you want to happen. -## Describe Alternatives You've Considered - +## Alternative Solutions A clear and concise description of any alternative solutions or features you've considered. +## Use Case +Describe how you would use this feature and why it would be valuable. + +## Code Example (Optional) +```python +# Example of how you envision using this feature +``` + ## Additional Context +Add any other context, mockups, or screenshots about the feature request here. -Add any other context or screenshots about the feature request here. +## Would You Like to Contribute? +- [ ] Yes, I would like to implement this feature +- [ ] No, I'm just suggesting the idea diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index 1b558dd..1b957aa 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -1,19 +1,26 @@ --- -name: Question +name: πŸ”§ Question about: Ask a question about this project -title: '' -labels: question +title: '[QUESTION]: ' +labels: ['question'] assignees: '' --- -## Your Question - -A clear and concise description of your question. +## Question +Your question here. Please be as specific as possible. ## Context +What are you trying to accomplish? What have you tried so far? -Provide any relevant context that might help answer your question. +## Relevant Code +```python +# Include any relevant code snippets here +``` -## What Have You Tried? +## Environment (if applicable) +- **OS**: +- **Python Version**: +- **Package Version**: -Describe what you've already tried or researched. +## Additional Information +Any other details that might help answer your question. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 065290e..62c8e2d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,30 +1,50 @@ ## Description - -A brief description of the changes in this PR. + ## Type of Change - -- [ ] Bug fix (non-breaking change which fixes an issue) -- [ ] New feature (non-breaking change which adds functionality) -- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) -- [ ] Documentation update -- [ ] Refactoring (no functional changes) + +- [ ] πŸ› Bug fix (non-breaking change which fixes an issue) +- [ ] ✨ New feature (non-breaking change which adds functionality) +- [ ] πŸ’₯ Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] πŸ“ Documentation update +- [ ] πŸ”§ Refactor (no functional changes) +- [ ] ⚑ Performance improvement +- [ ] πŸ§ͺ Test updates +- [ ] πŸ”’ Security fix ## Related Issues - -Closes #(issue number) - -## How Has This Been Tested? - -Describe the tests that you ran to verify your changes. + +Closes # + +## Changes Made + +- +- +- + +## Testing + +- [ ] I have added/updated tests that prove my fix is effective or that my feature works +- [ ] All existing tests pass locally with my changes +- [ ] I have tested this manually + +### Test Instructions +```bash +# Add instructions for testing your changes +``` ## Checklist - -- [ ] My code follows the style guidelines of this project -- [ ] I have performed a self-review of my own code + +- [ ] My code follows the project's style guidelines +- [ ] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings -- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have added tests that prove my changes work - [ ] New and existing unit tests pass locally with my changes -- [ ] Any dependent changes have been merged and published in downstream modules + +## Screenshots (if applicable) + + +## Additional Notes + diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6cbcb02..064913f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,19 +1,33 @@ version: 2 updates: + # Python dependencies - package-ecosystem: "pip" directory: "/" schedule: interval: "weekly" + day: "monday" + time: "09:00" + timezone: "Asia/Kolkata" open-pull-requests-limit: 10 + reviewers: + - "Coderooz" labels: - "dependencies" - "python" + commit-message: + prefix: "deps" + include: "scope" + # GitHub Actions - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" - open-pull-requests-limit: 5 + day: "monday" + time: "09:00" + timezone: "Asia/Kolkata" labels: - "dependencies" - - "github-actions" + - "ci-cd" + commit-message: + prefix: "ci" diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 0000000..2426085 --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,48 @@ +- name: "bug" + color: "d73a4a" + description: "Something isn't working" +- name: "enhancement" + color: "a2eeef" + description: "New feature or request" +- name: "documentation" + color: "0075ca" + description: "Improvements or additions to documentation" +- name: "good first issue" + color: "7057ff" + description: "Good for newcomers" +- name: "help wanted" + color: "008672" + description: "Extra attention is needed" +- name: "question" + color: "d876e3" + description: "Further information is requested" +- name: "wontfix" + color: "ffffff" + description: "This will not be worked on" +- name: "duplicate" + color: "cfd3d7" + description: "This issue or pull request already exists" +- name: "needs-triage" + color: "fbca04" + description: "Needs review and triaging" +- name: "in-progress" + color: "0e8a16" + description: "Work is currently being done on this" +- name: "testing" + color: "c2e0c6" + description: "Related to testing" +- name: "performance" + color: "016175" + description: "Performance improvements" +- name: "refactor" + color: "1d76db" + description: "Code refactoring" +- name: "security" + color: "ee0701" + description: "Security-related issues or fixes" +- name: "dependencies" + color: "0366d6" + description: "Pull requests that update a dependency file" +- name: "python" + color: "2b5797" + description: "Python-related issues" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0e59af6..6569245 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,39 +1,59 @@ -name: Build +name: Build and Package on: push: - branches: [main] + branches: [ main, master, develop ] pull_request: - branches: [main] + branches: [ main, master, develop ] + release: + types: [published] jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.12", "3.13"] steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install build tools - run: | - python -m pip install --upgrade pip - pip install build twine - - - name: Build package - run: python -m build - - - name: Check package - run: twine check dist/* - - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - with: - name: dist-${{ matrix.python-version }} - path: dist/ + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build wheel setuptools twine + + - name: Build package + run: | + python -m build + + - name: Check package + run: | + twine check dist/* + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: dist-packages + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + if: github.event_name == 'release' && github.event.action == 'published' + + steps: + - uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: dist-packages + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index aa261e8..8be634b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,40 +1,47 @@ -name: Lint +name: Python Lint on: push: - branches: [main, develop] + branches: [ main, master, develop ] pull_request: - branches: [main] + branches: [ main, master, develop ] jobs: lint: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.12", "3.13"] + python-version: ['3.12', '3.13'] steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install black isort flake8 mypy - pip install -e ".[dev]" - - - name: Check formatting with black - run: black --check . - - - name: Check import sorting with isort - run: isort --check-only . - - - name: Lint with flake8 - run: flake8 . - - - name: Type check with mypy - run: mypy . + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 black isort mypy + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Check formatting with Black + run: | + black --check --diff . + + - name: Check imports with isort + run: | + isort --check-only --diff . + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=120 --statistics + + - name: Type check with mypy + run: | + mypy --ignore-missing-imports . || true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e2a4c0..09887f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,40 +1,49 @@ -name: Test +name: Python Tests on: push: - branches: [main, develop] + branches: [ main, master, develop ] pull_request: - branches: [main] + branches: [ main, master, develop ] jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ["3.12", "3.13"] + python-version: ['3.12', '3.13'] steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" - - - name: Run tests - run: pytest - - - name: Run tests with coverage - run: pytest --cov=custom_functions --cov-report=xml - - - name: Upload coverage to Codecov - if: matrix.python-version == '3.12' - uses: codecov/codecov-action@v4 - with: - file: ./coverage.xml - fail_ci_if_error: false + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pytest-cov + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install -e ".[dev]" + + - name: Run tests with pytest + run: | + pytest --cov=custom_functions --cov-report=xml --cov-report=html + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + if: always() + with: + name: coverage-report-py${{ matrix.python-version }} + path: htmlcov/ diff --git a/.gitignore b/.gitignore index 294f135..972ead7 100644 --- a/.gitignore +++ b/.gitignore @@ -86,14 +86,6 @@ ipython_config.py # pipenv Pipfile.lock -# poetry -poetry.lock - -# pdm -.pdm.toml -.pdm-python -.pdm-build/ - # PEP 582 __pypackages__/ @@ -137,31 +129,30 @@ dmypy.json # Cython debug symbols cython_debug/ -# IDEs -.vscode/ +# PyCharm .idea/ -*.swp -*.swo -*~ -# OS -.DS_Store +# VS Code +.vscode/ + +# Thumbs.db (Windows) Thumbs.db -# Project-specific +# OS files +.DS_Store + +# Database files (project-specific) *.db *.sqlite -*.sqlite3 -logs/ -*.log -.env.local -.env.development.local -.env.test.local -.env.production.local - -# Workspace (governance - not committed) -.workspace/ -# MCP runtime (must stay in root per governance) -.mcp-runtime.json -*.project-mcp.json +# Test data +test_*.txt +test_*.csv +test_*.xlsx +test_*.pdf + +# Temporary files +*.tmp +*.temp +~$*.doc* +~$*.xls* diff --git a/.mcp-runtime.json b/.mcp-runtime.json new file mode 100644 index 0000000..6cad487 --- /dev/null +++ b/.mcp-runtime.json @@ -0,0 +1,10 @@ +{ + "port": 47000, + "pid": 8736, + "startedAt": 1778893041824, + "hostname": "Coderooz_PC", + "status": "running", + "project": "local-mcp-server", + "signature": "82660ef6-40f5-451a-9538-951e3ace798b", + "lastUpdated": "2026-05-16T01:25:16.828Z" +} diff --git a/.opencode/reference/PROJECT_REFERENCE_INDEX.md b/.opencode/reference/PROJECT_REFERENCE_INDEX.md deleted file mode 100644 index 1042a5a..0000000 --- a/.opencode/reference/PROJECT_REFERENCE_INDEX.md +++ /dev/null @@ -1,207 +0,0 @@ -# Project Reference Index (PRI) - -**Project:** custom_functions (My_simple_functions) -**Version:** 1.0.0 -**Last Updated:** 2026-09-07 -**Status:** Active - ---- - -## Project Identity - -| Field | Value | -|-------|-------| -| Name | custom_functions | -| Package Name | custom-functions | -| Language | Python 3.12+ | -| Type | Library (PyPI package) | -| License | MIT | -| Author | Ranit Saha (@coderooz) | -| Repository | https://github.com/coderooz/My_simple_functions | - ---- - -## Directory Structure - -``` -custom_functions/ -β”œβ”€β”€ .github/ # GitHub configuration -β”‚ β”œβ”€β”€ CODEOWNERS # Code ownership -β”‚ β”œβ”€β”€ dependabot.yml # Dependency updates -β”‚ β”œβ”€β”€ ISSUE_TEMPLATE/ # Issue templates -β”‚ β”‚ β”œβ”€β”€ bug_report.md -β”‚ β”‚ β”œβ”€β”€ feature_request.md -β”‚ β”‚ └── question.md -β”‚ β”œβ”€β”€ PULL_REQUEST_TEMPLATE.md -β”‚ └── workflows/ # CI/CD pipelines -β”‚ β”œβ”€β”€ build.yml -β”‚ β”œβ”€β”€ lint.yml -β”‚ └── test.yml -β”œβ”€β”€ .opencode/ # OpenCode configuration -β”‚ └── reference/ -β”‚ └── PROJECT_REFERENCE_INDEX.md # This file -β”œβ”€β”€ .workspace/ # Development artifacts (governance) -β”œβ”€β”€ DbHandler/ # Database handlers package -β”‚ β”œβ”€β”€ __init__.py -β”‚ β”œβ”€β”€ MySQLHandler.py # MySQL operations -β”‚ └── SqliteHandler.py # SQLite operations -β”œβ”€β”€ DataHandlers.py # Data manipulation utilities -β”œβ”€β”€ FileHandler.py # File I/O operations -β”œβ”€β”€ HtmlScraper.py # HTML scraping -β”œβ”€β”€ Requester.py # HTTP requests -β”œβ”€β”€ __init__.py # Package root -β”œβ”€β”€ setup.py # Legacy setup -β”œβ”€β”€ pyproject.toml # Modern Python config -β”œβ”€β”€ requirement.txt # Dependencies -β”œβ”€β”€ Readme.md # Documentation -β”œβ”€β”€ LICENSE # MIT License -β”œβ”€β”€ CHANGELOG.md # Version history -β”œβ”€β”€ CONTRIBUTING.md # Contribution guide -β”œβ”€β”€ CODE_OF_CONDUCT.md # Community guidelines -β”œβ”€β”€ SECURITY.md # Security policy -β”œβ”€β”€ .gitignore # Git ignore rules -β”œβ”€β”€ .gitattributes # Git attributes -└── .editorconfig # Editor config -``` - ---- - -## Module Reference - -### DataHandlers.py - -| Class/Function | Description | -|----------------|-------------| -| `DataHandler` | Timestamp formatting and data manipulation utilities | -| `DataHandler.timestamp()` | Format datetime with timezone support | -| `valreplace()` | Replace values in lists/dicts | -| `equalizer_dict()` | Equalize dictionary structures | - -### DbHandler/ (Package) - -#### SqliteHandler.py - -| Class | Description | -|-------|-------------| -| `SqliteHandler` | Full SQLite database management | -| `SqliteHandler.insert()` | Insert data into tables | -| `SqliteHandler.fetch()` | Query data with filtering | -| `SqliteHandler.createTb()` | Create tables | -| `SqliteHandler.update()` | Update records | -| `SqliteHandler.json_insert()` | Insert from JSON data | -| `SqliteHandler.csv_insert()` | Insert from CSV files | -| `SqliteHandler.export_data()` | Export to JSON/CSV | - -#### MySQLHandler.py - -| Class | Description | -|-------|-------------| -| `MySqlHandler` | MySQL database operations | -| `MySqlHandler.createTb()` | Create MySQL tables | -| `MySqlHandler.connect_db()` | Connect to databases | -| `MySqlHandler.execute()` | Execute MySQL queries | - -### FileHandler.py - -| Method | Description | -|--------|-------------| -| `FileHandler.read()` | Read file contents | -| `FileHandler.write()` | Append to file | -| `FileHandler.write_over()` | Overwrite file | -| `FileHandler.getFiles()` | List directory contents | -| `FileHandler.read_csv()` | Read CSV files | -| `FileHandler.write_csv()` | Write CSV files | -| `FileHandler.read_excel()` | Read Excel files | -| `FileHandler.write_excel()` | Write Excel files | -| `FileHandler.read_pdf()` | Extract PDF text | - -### Requester.py - -| Class | Description | -|-------|-------------| -| `Requester` | HTTP request handling | -| `Requester.get()` | GET requests | -| `Requester.post()` | POST requests | - -### HtmlScraper.py - -| Class | Description | -|-------|-------------| -| `HtmlScraper` | HTML parsing and scraping | - ---- - -## Dependencies - -| Package | Version | Purpose | -|---------|---------|---------| -| pytz | >=2023.3.0 | Timezone handling | -| requests | >=2.31.0 | HTTP requests | -| db-sqlite3 | >=0.0.1 | SQLite support | -| pandas | >=2.1.3 | Data manipulation | -| mysql-connector-python | >=2.2.9 | MySQL connectivity | -| openpyxl | >=3.1.2 | Excel file support | -| PyPDF2 | >=3.0.0 | PDF processing | - ---- - -## Development Dependencies - -| Package | Version | Purpose | -|---------|---------|---------| -| pytest | >=7.0.0 | Testing framework | -| pytest-cov | >=4.0.0 | Coverage reporting | -| black | >=23.0.0 | Code formatting | -| isort | >=5.0.0 | Import sorting | -| flake8 | >=6.0.0 | Linting | -| mypy | >=1.0.0 | Type checking | -| twine | >=4.0.0 | Package publishing | - ---- - -## CI/CD Workflows - -| Workflow | Trigger | Purpose | -|----------|---------|---------| -| lint.yml | push/PR to main | Code quality checks | -| test.yml | push/PR to main | Run test suite | -| build.yml | push/PR to main | Build and verify package | - ---- - -## Governance Compliance - -| Requirement | Status | -|-------------|--------| -| `.workspace/` directory | βœ… Present | -| `.gitignore` | βœ… Present | -| `.gitattributes` | βœ… Present | -| `.editorconfig` | βœ… Present | -| LICENSE | βœ… MIT | -| CONTRIBUTING.md | βœ… Present | -| CODE_OF_CONDUCT.md | βœ… Present | -| SECURITY.md | βœ… Present | -| CHANGELOG.md | βœ… Present | -| CI/CD workflows | βœ… Present | -| Issue templates | βœ… Present | -| PR template | βœ… Present | -| CODEOWNERS | βœ… Present | -| Dependabot | βœ… Present | -| PRI (this file) | βœ… Present | - ---- - -## Notes - -- `DbHandler` is a package (directory) in local project, not a single file as in GitHub upload -- `requirement.txt` is kept for backward compatibility; prefer `pyproject.toml` -- `.workspace/` contains development artifacts and is gitignored -- All governance requirements are satisfied - ---- - -## Revision History - -| Version | Date | Changes | -|---------|------|---------| -| 1.0.0 | 2026-09-07 | Initial PRI creation | diff --git a/AsyncHandler.py b/AsyncHandler.py new file mode 100644 index 0000000..66d2cc8 --- /dev/null +++ b/AsyncHandler.py @@ -0,0 +1,87 @@ +import asyncio, threading +import random + + +class AsyncThreadHandler: + def __init__(self): + self.tasks = [] + + def create_tasks(self, func): + pass + + + +class AsyncHandlerAio: + + def __init__(self) -> None: + self.tasks = [] + + def create_task(self, task_function, *args, **kwargs) -> None: + task = asyncio.create_task(task_function(*args, **kwargs)) + self.tasks.append(task) + + async def schedule_task(self, task_function, when, *args, **kwargs): + """ + Schedule an asynchronous task to be run at a specific time or at regular intervals. + + Parameters: + - task_function: the function to be run as an asynchronous task + - when: a float or int specifying the number of seconds in the future to run the task (for a single run) or the number of seconds between runs (for a recurring task) + - *args: positional arguments to be passed to the task function + - **kwargs: keyword arguments to be passed to the task function + + Returns: + - A Task object representing the scheduled task + """ + # Create a task using the provided task function and arguments + task = asyncio.create_task(task_function(*args, **kwargs)) + + # Schedule the task to be run at the specified time or interval + if when > 0: + # Schedule the task to run once in the future + asyncio.get_event_loop().call_later(when, task) + elif when < 0: + # Schedule the task to run repeatedly at a fixed interval + asyncio.get_event_loop().call_repeatedly(-when, task) + return task + + async def stop_tasks(self): + # Stop all the asynchronous tasks in the handler as before + for task in self.tasks: + task.cancel() + for task in self.running_tasks: + task.cancel() + await asyncio.gather(*(self.tasks + self.running_tasks), return_exceptions=True) + self.tasks = [] + self.running_tasks = [] + + async def run_task(self) -> None: + while self.tasks: + while len(self.running_tasks) < self.concurrency and self.tasks: + task = self.tasks.pop(0) + self.running_tasks.append(task) + task.add_done_callback(self.running_tasks.remove) + await asyncio.wait(self.running_tasks, return_when=asyncio.FIRST_COMPLETED) + + def get_status(self) -> dict: + """Get the status of the tasks being managed by the handler""" + return {'pending': len(self.tasks), 'running': len(self.running_tasks)} + +if __name__ == "__main__": + + async def greet(name): + print(f"Hello, {name}!") + sleep_time = random.uniform(0.5, 2.0) # Generate random sleep time between 0.5 and 2.0 seconds + await asyncio.sleep(sleep_time) + print(f"Goodbye, {name}!") + + async def main(): + # Create tasks for two concurrent greetings + task1 = asyncio.create_task(greet("Alice")) + task2 = asyncio.create_task(greet("Bob")) + + # Wait for both tasks to complete + await asyncio.gather(task1, task2) + + # Run the main coroutine + asyncio.run(main()) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6c828f..1f1424b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changelog +# CHANGELOG All notable changes to this project will be documented in this file. @@ -8,29 +8,69 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Git repository initialization -- Governance-compliant project structure -- `.workspace/` directory for development artifacts -- `.gitignore` for Python projects -- `.gitattributes` for cross-platform consistency -- `pyproject.toml` for modern Python packaging -- `CONTRIBUTING.md` guidelines -- `CODE_OF_CONDUCT.md` -- `SECURITY.md` policy -- `CHANGELOG.md` +- Initial release of Custom Functions package +- `DataHandler` class for data manipulation utilities +- `DbHandler` class for SQLite database operations +- `FileHandler` class for file I/O operations +- `Requester` class for HTTP requests ### Changed -- `DbHandler` converted from single file to package structure - - `DbHandler/SqliteHandler.py` - SQLite operations - - `DbHandler/MySQLHandler.py` - MySQL operations - - `DbHandler/__init__.py` - Package exports +- N/A + +### Deprecated +- N/A + +### Removed +- N/A + +### Fixed +- N/A + +### Security +- N/A + +--- -## [1.0.0] - 2024-01-01 +## [0.0.1] - 2024-XX-XX ### Added -- Initial release -- `DataHandler` class for timestamp formatting -- `DbHandler` class for database operations -- `FileHandler` class for file operations -- `Requester` class for HTTP requests -- `HtmlScraper` class for HTML scraping +- Initial project setup +- Core handler classes: + - `DataHandler`: Timestamp formatting and data utilities + - `DbHandler`: SQLite database CRUD operations, table management, and schema modifications + - `FileHandler`: File reading/writing, CSV/Excel/PDF handling + - `Requester`: HTTP request utilities +- Package distribution via `setup.py` +- MIT License + +--- + +## Template for Future Releases + +Copy and paste this template for new releases: + +## [VERSION] - DATE + +### Added +- + +### Changed +- + +### Deprecated +- + +### Removed +- + +### Fixed +- + +### Security +- + +--- + +**Author**: Ranit Saha +**Website**: [https://coderooz.in](https://coderooz.in) +**Contact**: [contact@coderooz.in](mailto:contact@coderooz.in) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 9ee532c..c186ff5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -9,24 +9,30 @@ identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + ## Our Standards -Examples of behavior that contributes to a positive environment: +Examples of behavior that contributes to a positive environment for our +community include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community -Examples of unacceptable behavior: +Examples of unacceptable behavior include: * The use of sexualized language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission +* Publishing others' private information, such as a physical or email + address, without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting @@ -37,21 +43,94 @@ acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + ## Scope This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at -contact@coderooz.in. +[contact@coderooz.in](mailto:contact@coderooz.in) or via the +[contact form](https://coderooz.in/contact?subject=[Code_of_Conduct_Violation]). All complaints will be reviewed and investigated promptly and fairly. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. + +--- + +**Project**: Custom Functions +**Author**: Ranit Saha +**Website**: [https://coderooz.in](https://coderooz.in) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e19d881..38b3fb4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,65 +1,173 @@ # Contributing to Custom Functions -Thank you for your interest in contributing to Custom Functions! This document provides guidelines and information about contributing to this project. +Thank you for your interest in contributing to **Custom Functions**! πŸŽ‰ + +This document provides guidelines and instructions for contributing. Please take a moment to read through it before making your first contribution. + +## Table of Contents + +- [Code of Conduct](#code-of-conduct) +- [Getting Started](#getting-started) +- [Development Setup](#development-setup) +- [How to Contribute](#how-to-contribute) +- [Pull Request Process](#pull-request-process) +- [Coding Standards](#coding-standards) +- [Testing Guidelines](#testing-guidelines) +- [Documentation](#documentation) +- [Commit Messages](#commit-messages) +- [Issue Reporting](#issue-reporting) +- [Community](#community) ## Code of Conduct -By participating in this project, you agree to abide by our [Code of Conduct](CODE_OF_CONDUCT.md). +By participating in this project, you agree to abide by our [Code of Conduct](CODE_OF_CONDUCT.md). Please read it before contributing. + +## Getting Started + +### Prerequisites + +- Python 3.12 or higher +- pip (Python package manager) +- Git + +### Development Setup + +1. **Fork the Repository** + - Click the "Fork" button at the top right of the repository page + - Clone your fork locally: + ```bash + git clone https://github.com/YOUR_USERNAME/My_simple_functions.git + cd My_simple_functions + ``` + +2. **Set Up Remote Upstream** + ```bash + git remote add upstream https://github.com/coderooz/My_simple_functions.git + ``` + +3. **Create a Virtual Environment** + ```bash + python -m venv venv + source venv/bin/activate # On Windows: venv\Scripts\activate + ``` + +4. **Install Dependencies** + ```bash + pip install -e ".[dev]" + pip install flake8 black isort pytest pytest-cov + ``` + +5. **Create a Branch** + ```bash + git checkout -b feature/your-feature-name + # or + git checkout -b fix/your-bug-fix + ``` ## How to Contribute -### Reporting Bugs +### Types of Contributions -Before creating bug reports, please check the existing issues to avoid duplicates. +We welcome various types of contributions: -When creating a bug report, please include: +- πŸ› **Bug Fixes**: Fix issues reported in the issue tracker +- ✨ **New Features**: Add new functionality or handlers +- πŸ“ **Documentation**: Improve README, docstrings, or comments +- πŸ§ͺ **Tests**: Add or improve test coverage +- ⚑ **Performance**: Optimize existing code +- πŸ”§ **Refactoring**: Improve code structure without changing behavior +- πŸ”’ **Security**: Fix security vulnerabilities -- A clear and descriptive title -- Steps to reproduce the issue -- Expected behavior -- Actual behavior -- Your environment (OS, Python version, etc.) +### Finding Issues to Work On -### Suggesting Features +- Look for issues labeled [`good first issue`](https://github.com/coderooz/My_simple_functions/labels/good%20first%20issue) for beginner-friendly tasks +- Check [`help wanted`](https://github.com/coderooz/My_simple_functions/labels/help%20wanted) for areas where we need assistance +- Always comment on an issue before starting work to avoid duplicate efforts -Feature suggestions are welcome! Please provide: +## Pull Request Process -- A clear description of the feature -- Use cases -- Any implementation ideas +1. **Ensure your code follows the coding standards** (see below) +2. **Add or update tests** for your changes +3. **Update documentation** if necessary +4. **Run the test suite** and ensure all tests pass +5. **Update CHANGELOG.md** with your changes +6. **Submit a Pull Request** using the [PR template](.github/PULL_REQUEST_TEMPLATE.md) -### Pull Requests +### PR Checklist -1. Fork the repository -2. Create a feature branch (`git checkout -b feature/amazing-feature`) -3. Make your changes -4. Run tests and linting -5. Commit your changes (`git commit -m 'feat: add amazing feature'`) -6. Push to the branch (`git push origin feature/amazing-feature`) -7. Open a Pull Request +- [ ] Code follows project style guidelines +- [ ] Self-review completed +- [ ] Code is commented where necessary +- [ ] Documentation updated +- [ ] No new warnings generated +- [ ] Tests added/updated and passing +- [ ] CHANGELOG.md updated -## Development Setup +### Review Process -### Prerequisites +- Project maintainers will review your PR +- Address any requested changes +- Once approved, a maintainer will merge your PR -- Python 3.12 or higher -- pip +## Coding Standards + +We use the following tools to maintain code quality: + +- **Black**: Code formatter +- **isort**: Import sorter +- **flake8**: Linter +- **mypy**: Type checker (optional) -### Setup +### Running Formatters and Linters ```bash -# Clone the repository -git clone https://github.com/coderooz/My_simple_functions.git -cd My_simple_functions +# Format code with Black +black . + +# Sort imports with isort +isort . -# Create virtual environment -python -m venv venv -source venv/bin/activate # Windows: venv\Scripts\activate +# Run flake8 linter +flake8 . + +# Run all checks +black . && isort . && flake8 . +``` -# Install in development mode -pip install -e ".[dev]" +### Python Style Guidelines + +- Follow [PEP 8](https://peps.python.org/pep-0008/) style guide +- Use type hints where possible +- Write descriptive variable and function names +- Keep functions focused and single-purpose +- Maximum line length: 120 characters + +### Example Function Structure + +```python +def example_function(param1: str, param2: int = 10) -> dict: + """Brief description of what the function does. + + Args: + param1: Description of param1 + param2: Description of param2 (default: 10) + + Returns: + Description of return value + + Raises: + ValueError: When param1 is empty + """ + if not param1: + raise ValueError("param1 cannot be empty") + + # Implementation + result = {"key": param1, "count": param2} + return result ``` +## Testing Guidelines + ### Running Tests ```bash @@ -70,45 +178,131 @@ pytest pytest --cov=custom_functions --cov-report=html # Run specific test file -pytest tests/test_datahandler.py -v +pytest tests/test_datahandler.py + +# Run with verbose output +pytest -v ``` -### Code Quality +### Writing Tests -```bash -# Format code -black . +- Place tests in the `tests/` directory +- Name test files as `test_.py` +- Use descriptive test function names: `test__` +- Test both normal cases and edge cases +- Mock external dependencies when appropriate -# Sort imports -isort . +### Example Test -# Lint -flake8 . +```python +import pytest +from custom_functions.DataHandler import DataHandler -# Type check -mypy . -``` +def test_timestamp_returns_string(): + """Test that timestamp returns a formatted string.""" + result = DataHandler.timestamp() + assert isinstance(result, str) -## Coding Standards +def test_timestamp_custom_format(): + """Test timestamp with custom format.""" + result = DataHandler.timestamp(format="%Y-%m-%d") + assert len(result) == 10 +``` -- Follow PEP 8 style guide -- Use type hints for all function parameters and return values -- Write docstrings for all public functions and classes -- Keep functions small and focused -- Write tests for new functionality +## Documentation + +Good documentation is essential. When contributing: + +- Update README.md if adding new features +- Add docstrings to all public functions and classes +- Update inline comments for complex logic +- Include usage examples in docstrings + +### Docstring Format + +We use Google-style docstrings: + +```python +def my_function(param1: str) -> bool: + """One-line summary. + + Detailed description of what the function does, + its behavior, and any important notes. + + Args: + param1: Description of parameter + + Returns: + Description of return value + + Raises: + ExceptionType: When condition occurs + """ +``` ## Commit Messages -Follow [Conventional Commits](https://www.conventionalcommits.org/): +We follow [Conventional Commits](https://www.conventionalcommits.org/) specification: + +``` +(): + +[optional body] + +[optional footer(s)] +``` + +### Types + +- `feat`: New feature +- `fix`: Bug fix +- `docs`: Documentation changes +- `style`: Code style changes (formatting, etc.) +- `refactor`: Code refactoring +- `test`: Adding or updating tests +- `chore`: Maintenance tasks + +### Examples + +``` +feat(DataHandler): add JSON parsing utility +fix(FileHandler): handle empty file edge case +docs: update README with new installation instructions +test(DbHandler): add tests for createTb method +``` + +## Issue Reporting + +### Before Creating an Issue + +1. Check existing issues to avoid duplicates +2. Use the appropriate issue template +3. Provide as much context as possible + +### Issue Templates + +- πŸ› [Bug Report](.github/ISSUE_TEMPLATE/bug_report.md) +- ✨ [Feature Request](.github/ISSUE_TEMPLATE/feature_request.md) +- πŸ“ [Documentation Improvement](.github/ISSUE_TEMPLATE/docs_improvement.md) +- πŸ”§ [Question](.github/ISSUE_TEMPLATE/question.md) + +## Community + +- **Website**: [https://coderooz.in](https://coderooz.in) +- **Contact**: [https://coderooz.in/contact](https://coderooz.in/contact?subject=[message_subject]&message=[message]) +- **Email**: [contact@coderooz.in](mailto:contact@coderooz.in) + +## Recognition + +All contributors will be recognized in: + +- The CHANGELOG.md +- The README.md contributors section +- GitHub's contributors page -- `feat:` for new features -- `fix:` for bug fixes -- `docs:` for documentation changes -- `style:` for formatting changes -- `refactor:` for code refactoring -- `test:` for adding tests -- `chore:` for maintenance tasks +Thank you for contributing to Custom Functions! πŸ™ -## Questions? +--- -If you have questions about contributing, please open an issue with the "question" label. +**Author**: Ranit Saha +**Website**: [https://coderooz.in](https://coderooz.in) diff --git a/DataHandlers.py b/DataHandlers.py index 6e983ef..018bf38 100644 --- a/DataHandlers.py +++ b/DataHandlers.py @@ -1,5 +1,5 @@ from typing import Dict, List, Optional, Union -import time, json, os, pytz, re, functools +import time, json, os, pytz, re from dateutil.relativedelta import relativedelta from datetime import datetime, time, timedelta from collections.abc import KeysView, ValuesView @@ -7,7 +7,7 @@ #### For Time Based Methods #### -def generate_time_intervals(start_time, duration_split:int=1, duration_format:str='d', output_format:str='human', datetime_format="%Y-%m-%d %H:%M:%S", end_time=None, excludeDate=None, excludeDays:list=[]): +def generate_time_intervals(start_time, duration_split:int=1, duration_format:str='d', output_format:str='human', datetime_format="%Y-%m-%d %H:%M:%S", end_time=None, excludeDate=None): """ Generate a list of time intervals based on the provided parameters. @@ -24,14 +24,6 @@ def generate_time_intervals(start_time, duration_split:int=1, duration_format:st - end_time (datetime, str, optional): The end point for generating time intervals. If not provided, the default is the current time. - excludeDate (str, datetime, list[str], optional): The date's that are to be excluded. - - excludeDays (list[int], optional): Takes the days as numbers. Will exclude days of the week. Values for: - - 0: Sunday - - 1: Monday - - 2: Tuesday - - 3: Wednesday - - 4: Thursday - - 5: Friday - - 6: Saturday Returns: - time_intervals (list): A list of generated time intervals based on the provided parameters. @@ -65,7 +57,6 @@ def generate_time_intervals(start_time, duration_split:int=1, duration_format:st time_intervals = [i for i in time_intervals if i not in excludeDate] return time_intervals -@functools.cache def timestamp(given_time, format= "%Y-%m-%d %H:%M:%S", time_zone=None, normalize:str='sec'): """ timestamp() @@ -86,7 +77,6 @@ def timestamp(given_time, format= "%Y-%m-%d %H:%M:%S", time_zone=None, normalize return time.strftime(format, time.localtime(given_time)) else: raise ValueError('The parameter passed in given_time is invalid. Please provide a valid data') -@functools.cache def convert_timestamp(timestamp_ms, normalization='sec'): """ Converts the unix timestamp to the desired format like second, minute or hour.""" ty = type(timestamp_ms) @@ -97,7 +87,6 @@ def convert_timestamp(timestamp_ms, normalization='sec'): elif ty == list: return [convert_timestamp(i,normalization) for i in timestamp_ms] else: raise TypeError("Argument type passed is not valid.") -@functools.cache def future_timestamp(interval, unit, time_zone='UTC'): """ Calculate a future Unix timestamp based on the provided interval and unit. @@ -168,7 +157,6 @@ def get_previous_day(date:str='', format:str='%d-%m-%Y', numberof_days:int=1): date = date - timedelta(days=numberof_days) return date.strftime(format) -@functools.cache def identify_date_format(date_str:str): """ identify_date_format() @@ -209,7 +197,6 @@ def dateFormat(date:str, format1:str, format2:str = '')->(str|datetime): #### For Dict Based Methods #### -@functools.cache def equalizer_dict(data:List[dict], value='')->list: """ Make dictionaries equal by filling in missing keys and values. @@ -249,7 +236,18 @@ def equalizer_dict(data:List[dict], value='')->list: elif isinstance(lo, (int, float)): data2[t].append(0) return dlist_dict(data2) -@functools.cache +def dict_dimenstion_flatener(data:dict, catg): + """ + dict_dimenstion_flatener() + -------------------------- + This method is used to reducing a multi-dimention dict into a single dimention dict. + + Parameter: + - data (dict): takes the dict that is to made into a single dimention. + + """ + pass + def dict_lister(data:list, opt:list=None)->dict: """ Converts a list of dictionaries with list-type values into a dictionary with lists. @@ -277,7 +275,6 @@ def dict_lister(data:list, opt:list=None)->dict: if len(opt) == 1 and len(opt)!=None: return t[opt[0]] return t -@functools.cache def dict_filter(data, filter:list, catg=1)->dict[any]|list[dict[any]]: """ This function checks if a parameter exists in the dictionay. @@ -325,8 +322,7 @@ def flatten_dict(d, parent_key='', sep='_', catg:int=0): elif catg == 1: pass else: raise ValueError('The value passed into the parameter `catg` is not accepted. The only value accepted are `0` & `1`.') - -@functools.cache + def dict_reoganize(data, pattern:list): """ dict_reoganize() @@ -353,7 +349,7 @@ def dict_reoganize(data, pattern:list): raise ValueError('The data passed in the data parameter is invalid. The paramerter only accepts either dict or a list of dict.') #### For List Based Methods #### -@functools.cache + def dlist_dict(my_dict:dict, keys:list=[]) -> list: """ @@ -398,7 +394,6 @@ def dlist_dict(my_dict:dict, keys:list=[]) -> list: result_list = [{k: v[i] for k, v in my_dict.items() if k in keys} for i in range(len(list(my_dict.values())[0]))] return result_list -@functools.cache def list_dlist(data, keys): """ This method list_dlist arranged the list list i.e.([['a','n'],['b','o']) into {key1:['a','b'], key2:['n','o']} @@ -450,7 +445,6 @@ def decode_json(data): except: return None -@functools.cache def json_parser(data:dict, pathway): """ This function is designed to parse data in a JSON/dictionary structure based on a specified pathway. @@ -600,7 +594,6 @@ def check_value_exists(value, param): elif isinstance(param, (set, str)): return value in param else: return value in list(param) -@functools.cache def valreplace(data, target:str, replace:str, keyTy:bool=False): """ This method/function is used for replaceing certain or wanted values in the given subject data. @@ -626,7 +619,6 @@ def valreplace(data, target:str, replace:str, keyTy:bool=False): else: return data.replace(target, replace) else: raise ValueError(f'This method only expects list, dict or a string. Not {type(data)}') -@functools.cache def space_remover(data): """ This method is usd for removing any spaces in a string that is in front or behind the string. @@ -640,10 +632,9 @@ def space_remover(data): return data.strip() else: return data -@functools.cache def setNum(data): """ - This method is used for making any possible number or float that is in a string format turn into one. + This metod is used for making any possible number or float that is in a string format turn into one. """ if isinstance(data, dict): return {k.strip(): setNum(v) for k,v in data.items()} elif isinstance(data, list): return [setNum(i) for i in data] @@ -658,20 +649,9 @@ def setNum(data): else: return data -def setStr(data): - if isinstance(data, (list, tuple, KeysView, ValuesView)): return [setStr(i) for i in data] - elif isinstance(data, dict): return {k: setStr(i) for k,i in data.items()} - else: return f'{data}' - def get_similarities(list1, list2): """ This function returns the similarity between two lists and returns the simialar. """ return list(set(list1).intersection(set(list2))) -def checkTimeRun(func): - import time - start_time = time.time() - end_time = time.time() - - \ No newline at end of file diff --git a/DbHandler/SqliteHandler.py b/DbHandler.py similarity index 83% rename from DbHandler/SqliteHandler.py rename to DbHandler.py index 1d9b756..e5eef9f 100644 --- a/DbHandler/SqliteHandler.py +++ b/DbHandler.py @@ -1,17 +1,15 @@ import sqlite3, csv, json import pandas as pd from DataHandlers import valreplace, equalizer_dict -from FileHandler import getExtention, read, fileExists, write, delFiles +from FileHandler import getExtention, read, fileExists, write, write_csv from collections.abc import KeysView, ValuesView +import mysql.connector as myqC from datetime import datetime from typing import List, Union, Dict -import logging -logging.basicConfig(filename='./error.log', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') - class SqliteHandler(): """ - The `SqliteHandler` class simplifies interactions with SQLite3 databases in Python, offering a dynamic and efficient approach. It is designed to accelerate the development of database-related projects using Python's SQLite3 module. + The `DbSqliteHandler` class simplifies interactions with SQLite3 databases in Python, offering a dynamic and efficient approach. It is designed to accelerate the development of database-related projects using Python's SQLite3 module. Features: - Insert Data (insert) @@ -80,8 +78,6 @@ class SqliteHandler(): `DbSqliteHandler` offers a flexible and efficient way to interact with SQLite3 databases in Python, simplifying database-related tasks and enhancing the productivity of your projects. """ - # basic and startup operations - def __init__(self, dbname, dbPath:str='.', json_import:bool=False, default_timeout:int=5000): """ Initializes the DbSqliteHandler instance. @@ -90,38 +86,23 @@ def __init__(self, dbname, dbPath:str='.', json_import:bool=False, default_timeo - `dbname` (str): The name of the database. - `dbPath` (str, optional): The path to the database directory. Default is None. """ - self.db_init, self.db_conn = None, None - self.dbName, self.dbPath = dbname, dbPath + self.db_init = None + self.db_conn = None + self.dbName = dbname + self.dbPath = dbPath self.dbFullPath = self.dbPath+'/'+self.dbName ext = getExtention(self.dbFullPath) if json_import and ext == 'json': self.load_dbJson(self.dbFullPath, True) elif ext == 'db': - self._connect() + self.db_init = sqlite3.connect(self.dbFullPath) + self.db_conn = self.db_init.cursor() else: - logging.error(f:='File type error! only excepts json file containing dbcreating data or the db path.') - raise TypeError(f) + raise TypeError('File type error! only excepts json file containing dbcreating data or the db path.') self.DbtimeOut(default_timeout) - def _connect(self)->None: - """ - _connect(): - ----------- - For checking and creatin datbase connections. - """ - if self.db_conn: - print("Already connected to the database.") - return None - try: - self.db_init = sqlite3.connect(self.dbFullPath) - self.db_conn = self.db_init.cursor() - - except sqlite3.Error as e: - logging.error(er:=f"Error connecting to database: {e}") - print(er) - def execute(self, query, data:list=[], multi: bool = False, auto_commit=True): """ execute() @@ -153,12 +134,86 @@ def execute(self, query, data:list=[], multi: bool = False, auto_commit=True): self.db_init.commit() return result except sqlite3.Error as e: - logging.error('Sqlite3 Execution Error (%s) on query(%s)', e, query) print("SQLite error:", e) if auto_commit: self.db_init.rollback() return None - # --- Data handleing / CURD Operations --- # + # --- Data handleing --- # + + def insert(self, table:str, columns, values, createTb:bool=False): + """ + Inserts data into the specified table. + + Parameters: + - `table` (str): The name of the table to insert data into. + - `columns` (str, list, KeysView): Comma-separated column names. + - `values` (list): List of values to be inserted. + - `createTb` (bool): This will create a table in the db if not available. Default is `False`. + + Returns: + sqlite3.Cursor: The result of the executed query. + """ + if createTb==True and self.getTb(table_name=table) == False: self.createTb(tbName=table, columns=columns, primary_key='id') + k=False + if isinstance(columns, (KeysView, list, tuple, ValuesView)): + keys = ['INDEX','KEY','SELECT','INSERT','UPDATE','DELETE','FROM','WHERE','JOIN','INNER','LEFT','RIGHT','GROUP BY','ORDER BY','AS','COUNT','SUM','MAX','MIN','AVG','DISTINCT','AND','OR','NOT','BETWEEN','LIKE','IN','NULL','TRUE','FALSE','TOP','LIMIT','OFFSET'] + for k in keys: + columns = valreplace(columns, k, '_'+k.upper(), 1) # type: ignore + columns = valreplace(columns, k.lower(), '_'+k.lower(), 1) # type: ignore + columns = ','.join(columns) + + if isinstance(values, (KeysView, list, tuple, ValuesView)): + value = [] + for val in values: + if isinstance(val, (KeysView, list, tuple, ValuesView)): + value.append(str(tuple(val))) + + elif isinstance(val, str): + value = str(tuple(values)) + return self.execute(f'INSERT INTO {table} ({columns}) VALUES {value};') + value = ','.join(value) + k = self.execute(f'INSERT INTO {table} ({columns}) VALUES {value} ;') + elif isinstance(values, str): + k = self.execute(f'INSERT INTO {table} ({columns}) VALUES ({values});') + else: raise ValueError('Invalid input format! Please provide a valid set of values.') + + def json_insert(self, table_name:str, data, createTb:bool=False, ifExist:str='')->None: + ''' + JSON_INSERT() + ------------- + + This method is used to insert data into the table using json format data. + Parameters: + - `table_name` (str):This parametere of the method tales the name of the table in which the data is to inserted. + - `data` (list|dict): This parameter takes the data either in dict format or a list containing dicts if multiple entries are to add. + - `createTb` (bool): .. + - `ifExist` (list): A list of fields that should exist before inserting the record. If any field does not + - return None + ''' + + table_column = self.getColumnNames(table_name) + + if isinstance(data, dict): + col, query = [], [] + for k,i in data.items(): + if k in table_column and isinstance(i, (str, float, int)): + col.append(k) + query.append(i) + elif isinstance(data, list) and len(data) > 0: + data = equalizer_dict(data) + col = data[0].keys() + query = [list(i.values()) for i in data] + else:raise ValueError('The data type passed is invald. The data paramerter takes a dict or a list of dict.') + + col = list(col) + if ifExist!='': + check_data = self.fetch_unique(table_name, ifExist) + check_data = {ifExist: check_data} if isinstance(check_data, list) else check_data + for k,v in check_data.items(): + col_idx = col.index(k) + query = [val for val in query if val[col_idx] not in v] + + self.insert(table_name, col, query, createTb) def update(self, table:str, updatedata, condition:str=''): """ @@ -194,7 +249,7 @@ def update(self, table:str, updatedata, condition:str=''): if t : return True return False - def fetch(self, table:str, columns:str='*', query: str|list[str]|dict[str, int,str,float]='', limit:int=0, Offset:int=0, fetchAll:bool=True, assc:str='', desc:str='', useIndex:str|None=None, detailed:bool=True, fetchQuery:bool=False)->list[str, float, int]: + def fetch(self, table:str, columns:str='*', query: str|list[str]='', limit:int=0, Offset:int=0, fetchAll:bool=True, assc:str='', desc:str='', detailed:bool=True)->list[str, float, int]: """ fetch() ------- @@ -224,46 +279,41 @@ def fetch(self, table:str, columns:str='*', query: str|list[str]|dict[str, int,s ``` """ - if self.getTb(table) == False: raise ValueError(f'The table({table}) is not present in the database.') - - if isinstance(query, list)and len(query) > 0: - query:str= ' AND '.join([i for i in query if i!='']) + if self.getTb(table) == False: + raise ValueError(f'The table({table}) is not present in the database.') + + if isinstance(query, str) and query!='': + query = f' WHERE {query}' + elif isinstance(query, list)and len(query) > 0: + query = ' WHERE ' + ' AND '.join(query) elif isinstance(query, dict)and len(query.keys()) > 0: - query:str= ' AND '.join([f"{k}='{v}'" for k,v in query.items() if v!=None or v!='']) - - query:str= f' WHERE {query}' if query!='' else '' + query = ' WHERE ' + ' AND '.join([f"{k}='{v}'" for k,v in query.items() if v!=None or v!='']) + order:str = '' if desc != '' or assc != '': col:list= self.getColumnNames(table) ord:list = [] - if assc!='': ord.append(f'{assc} ASC' if ',' not in assc else ','.join(f'{i} ASC' for i in assc.split(',') if i in col)) - if desc!='': ord.append(f'{desc} DESC' if ',' not in assc else ','.join(f'{i} DESC' for i in desc.split(',') if i in col)) + if assc!='' and assc in col: ord.append(f'{assc} ASC') + if desc!='' and desc in col: ord.append(f'{desc} DESC') order = ' ORDER BY ' + ', '.join(ord) if limit > 0: limit:str = f' LIMIT {str(limit)},{str(Offset)}' if Offset != 0 and Offset > limit else f' LIMIT {str(limit)}' else: limit:str = '' - - useidx:str = f' INDEXED BY {useIndex}' if useIndex!=None else '' - sql_query=f"SELECT {columns} FROM {table}{useidx}{query}{order}{limit};" - if fetchQuery: return sql_query - ret = self.execute(sql_query) + query = f"SELECT {columns} FROM {table}{query}{order}{limit};" + ret = self.execute(query) try: if ret is not None: if detailed==False: if fetchAll: return ret.fetchall() else: return ret.fetchone() else: - data:list = [] col = [column[0] for column in ret.description] if ret.description else [] - if fetchAll:data = [dict(zip(col, row)) for row in ret.fetchall()] - elif ret.fetchone() is not None: data=dict(zip(col, ret.fetchone())) - logging.info('Fetched Data on quering("%s") successfully', sql_query) - return data + if fetchAll: return [dict(zip(col, row)) for row in ret.fetchall()] + elif ret.fetchone() is not None: return dict(zip(col, ret.fetchone())) else: return [] except Exception as e: - logging.error('FETCH Func Error(%e)', sql_query) print(e) return [] @@ -297,87 +347,6 @@ def getTbData(self, table_name:str,columns:str='*',query:str='', limit:int=0, of print(f'The table(`{table_name}`) is empty with no data.') return False - def insert(self, table:str, columns, values, createTb:bool=False): - """ - Inserts data into the specified table. - - Parameters: - - `table` (str): The name of the table to insert data into. - - `columns` (str, list, KeysView): Comma-separated column names. - - `values` (list): List of values to be inserted. - - `createTb` (bool): This will create a table in the db if not available. Default is `False`. - - Returns: - sqlite3.Cursor: The result of the executed query. - """ - if createTb==True and self.getTb(table_name=table) == False: self.createTb(tbName=table, columns=columns, primary_key='id') - k=False - if isinstance(columns, (KeysView, list, tuple, ValuesView)): - keys = ['INDEX','KEY','SELECT','INSERT','UPDATE','DELETE','FROM','WHERE','JOIN','INNER','LEFT','RIGHT','GROUP BY','ORDER BY','AS','COUNT','SUM','MAX','MIN','AVG','DISTINCT','AND','OR','NOT','BETWEEN','LIKE','IN','NULL','TRUE','FALSE','TOP','LIMIT','OFFSET'] - for k in keys: - columns = valreplace(columns, k, '_'+k.upper(), 1) # type: ignore - columns = valreplace(columns, k.lower(), '_'+k.lower(), 1) # type: ignore - columns = ','.join(columns) - - if isinstance(values, (KeysView, list, tuple, ValuesView)): - value = [] - for val in values: - if isinstance(val, (KeysView, list, tuple, ValuesView)): - value.append(str(tuple(val))) - - elif isinstance(val, str): - value = str(tuple(values)) - return self.execute(f'INSERT INTO {table} ({columns}) VALUES {value};') - value = ','.join(value) - k = self.execute(f'INSERT INTO {table} ({columns}) VALUES {value} ;') - elif isinstance(values, str): - k = self.execute(f'INSERT INTO {table} ({columns}) VALUES ({values});') - else: - logging.error(f:='INSERT func Error: Invalid input format! Please provide a valid set of values.') - raise ValueError(f) - - def json_insert(self, table_name:str, data, createTb:bool=False, ifExist:str='')->None: - """ - JSON_INSERT() - ------------- - - This method is used to insert data into the table using json format data. - Parameters: - - `table_name` (str):This parametere of the method tales the name of the table in which the data is to inserted. - - `data` (list|dict): This parameter takes the data either in dict format or a list containing dicts if multiple entries are to add. - - `createTb` (bool): .. - - `ifExist` (list): A list of fields that should exist before inserting the record. If any field does not - - return None - """ - - table_column = self.getColumnNames(table_name) - col,query = [], [] - if isinstance(data, dict): - col = data.keys() - query = list(data.values()) - elif isinstance(data, list) and len(data) > 0: - data = equalizer_dict(data) - col = data[0].keys() - query = [list(i.values()) for i in data] - else:raise ValueError('The data type passed is invald. The data paramerter takes a dict or a list of dict.') - col = list(col) - self.insert(table_name, col, query, createTb) - - def identify_cols(self, data:dict[str, str|float|int]): - """ - identify_cols() - --------------- - Identifies the data passed into the data parameter to select the table column types. - """ - ts = [] - for k,v in data.items(): - table = f'{k} ' - if isinstance(v, str): table+='TEXT' - elif isinstance(v, int): table+='INTEGER' - elif isinstance(v, float): table+='REAL' - ts.append(table) - return ts - ### Table work/ altering related method. ### def rearrange_table(self, table:str, orderOf:str, orderBy:str='asc', backup:bool=True)->None: @@ -437,49 +406,21 @@ def load_dbJson(self, data=None, fileName:bool=False)->None: else: raise ValueError('Check the value given passed as arguments.') - def export_data(self, catg:str='json', tableName:str='', remove:bool=True)->None: - """ - export_data(): - ------------- - Exports the data of the database in the desired file format. The exported file will be created in the same directory in which the databse is located. - Prameters: - - catg str: Takes the file type in which the data is to be exported. default is `'json'`.Values accepted are: - - `json` - - `sql` - - tableName str: to specify any specific table name that is to exported. - - remove bool: To specify for previously created file to be deleted or not. Default is `True`. - """ - if tableName=='': - delFiles(f'{self.dbPath}/{self.dbName.replace('.','_')}.{catg}') - for i in self.getTb(): - try: - self.export_data(catg, i, False) - print(i) - except Exception as e: logging.error('File Export Error (%e)') - else: - filePath = f'{self.dbPath}/{self.dbName.replace('.','_')}.{catg}' - if remove: delFiles(filePath) - otherPar:dict = {} - if catg=='json': - data = {'created_on': str(datetime.now().strftime('%d-%m-%Y %H:%M:%S %p')),'data':[], **self.get_info(tableName)} - k=self.get_info(tableName) - data['data'] = self.fetch(tb, detailed=False) if k['rows'] > 0 else [] - elif catg=='xls': - pass - elif catg == 'csv': - pass - elif catg == 'sql': - tb = self.get_info(tableName) - data:list= [] - columns = ','.join([f'`{i.split(' ')[0]}`' for i in tb['column_names']]) - values = ','.join([str(tuple(i)) for i in self.fetch(tableName, detailed=False)]) - data.append(f"CREATE TABLE `{tb['table_name']}` {tuple(tb['column_names'])} ENGINE = \'InnoDb\';") - # data.append(f'INSERT INTO `{tb['table_name']}` {columns} VALUES {values};') - otherPar['separator'] = '\n' - else: - logging.error('File Export Error (%e)', 'The type of file given is not accepted.') - raise ValueError('The type of file given is not accepted.') - write(filePath, data, **otherPar) + def export_data(self, catg:str='json', tableName:str=''): + if catg=='json': + data = {'db_name': self.dbName,'tables':[],'created_on': str(datetime.now().strftime('%d-%m-%Y %H:%M:%S %p'))} + for tb in self.getTb(): + k=self.get_info(tb) + if k['rows'] > 0: + k['data'] = self.fetch(tb, detailed=False) + data['tables'].append(k) + write(f'{self.dbPath}/{self.dbName.replace('.','_')}.json', data, emptyPervious=True) + elif catg=='xls': + pass + # write(f'{self.dbPath}/{self.dbName.replace('.','_')}.csv', data, emptyPervious=True) + elif catg == 'csv': + pass + else: raise ValueError('The type of file given is not accepted.') def getCount(self, table_name:str, columns:str='*', query:str='')->int: """ @@ -550,7 +491,7 @@ def csv_insert(self, table_name:str, csv_file_path:str): print(f"Error: {e}") return False - def get_excel(self, tbName:str='', columns:str='*', query='', fetchAll:bool=True, desc:str='', fileName:str='', filePath:str='.')->bool: + def get_excel(self, tbName:str='', columns:str='*', query:str='', fetchAll:bool=True, desc:str='', fileName:str='', filePath:str='.'): """ This method get the specified table and saves the data in the file @@ -567,17 +508,15 @@ def get_excel(self, tbName:str='', columns:str='*', query='', fetchAll:bool=Tru file: Returns a saved file. """ try: + data = self.getTbData(tbName,columns,query,fetchAll, desc) fileName = fileName if fileName!=None else f'{tbName}.csv' fileName = f'{filePath}/{fileName}' - query = self.fetch(tbName, columns, query=query, fetchAll=fetchAll, desc=desc, fetchQuery=True) - data = pd.read_sql_query(query, self.db_init) data.to_csv(fileName, index=False) - logging.error(f'SQLITE DB Table to EXCEL convertion success({fileName})') - return True + return 1 + except Exception as e: - logging.error(f'SQLITE DB Table to EXCEL convertion error ({e})') print(e) - return False + return 0 def beginTransaction(self): """Begin a transaction.""" @@ -945,6 +884,115 @@ def close_connection(self, mesg=None): if mesg!=None: print(mesg) - def __exit__(self, exc_type, exc_value, traceback): - self.close_connection() +class MySqlHandler(): + + def __init__(self, host:str, user:str, password:str, dataBase:str=""): + try: + self.dbConn = myqC.connect(host=host, user=user, passwd=password) + self.cursor = self.dbConn.cursor() + except myqC.Error as err: + if err.errno == myqC.errorcode.ER_ACCESS_DENIED_ERROR: + return 'Error Password!' + else: return err.errno + + + def createTb(self, tableName, columns, Engine:str='InnoDb', tableComment:str=''): + """ + """ + if self.getTables(tableName): raise ValueError(f'This table [{tableName}] already exist!') + column = ','.join(columns) + + query = f"CREATE TABLE `{tableName}` ({column}) ENGINE = \'{Engine}'" + if tableComment != '': + query += f" COMMENT = '{tableComment}'" + + query+= ';' + self.cursor.execute(query) + + def getTables(self, table:str=''): + """ + The method is to get the list of tables form the database. + """ + self.cursor.execute("SHOW TABLES") + tbList = [i[0] for i in self.cursor.fetchall()] + if table!='': return table in tbList + return tbList + + def addIndex(self, tableName:str, index_name:str, columns:list): + """ + This method is to add index to the tables. + """ + + columns = ','.join(columns) + query = f'`{index_name}` ({columns})' + self.alterTb(tableName, 'ADD UNIQUE', query) + + def alterTb(self, tableName:str, catg:str, query:str): + """ + This method is to used to alter tables in the database. + """ + self.cursor.execute(f"ALTER TABLE `{tableName}` {catg} {query};") + def cleanTb(self, tableName:str): + """ + This method is to clean the table i.e. it will delete all the data from the table. + """ + if self.getTables(tableName): + self.cursor.execute(f'TRUNCATE TABLE `{tableName}`;') + + def delTb(self, tableName:str): + """This method is to delete the specified table.""" + if self.getTables(tableName): + self.cursor.execute(f'DROP TABLE `{tableName}`;') + + def connect_db(self, dataBase:str, create_db:bool=True): + """ + The task of this mathod is to create new databases in the server. + """ + if self.getDbList(dataBase)==False and create_db: + self.createDb(dataBase) + self.dbConn.database = dataBase + + def createDb(self, dataBase:str): + """ + This method is to crreate a new dataBase. + """ + try: + if self.getDbList(dataBase) == True: ValueError('database already exists.') + return self.execute(f'CREATE DATABASE {dataBase}') + except Exception as e: return e + + def getDbList(self, present:str=''): + """ + This method gets the list of databases present in the server. + + """ + self.execute("SHOW DATABASES") + dbList = [i[0] for i in self.cursor.fetchall()] + if present != '': return present in dbList + return dbList + + def execute(self, query:str): + """ + This method is to execute the mysql queries. + """ + try: + return self.cursor.execute(query) + except: + pass + + def delDb(self, dataBase:str): + """ + This method is to delete dataBases. + """ + if self.getDbList(dataBase): + self.execute(f'DROP DATABASE {dataBase}') + else: + raise ValueError('Database does not exist!') + + def close_connection(self): + """ + This method is to close the connection. + """ + self.cursor.close() + self.dbConn.close() \ No newline at end of file diff --git a/DbHandler/MySQLHandler.py b/DbHandler/MySQLHandler.py deleted file mode 100644 index b14628c..0000000 --- a/DbHandler/MySQLHandler.py +++ /dev/null @@ -1,270 +0,0 @@ -import mysql.connector as myqC -from collections.abc import KeysView, ValuesView -from datetime import datetime -from typing import List, Union, Dict -import logging -from DataHandlers import equalizer_dict,setStr - -logging.basicConfig(filename='./error/mysqli_db.log', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') - - -class MySqlHandler(): - - def __init__(self, host:str, user:str, password:str, dataBase:str=""): - try: - self.dbConn = myqC.connect(host=host, user=user, passwd=password) - self.cursor = self.dbConn.cursor() - except myqC.Error as err: - if err.errno == myqC.errorcode.ER_ACCESS_DENIED_ERROR: - return 'Error Password!' - else: return err.errno - - self.database = None - if dataBase!="": self.connect_db(dataBase) - - - def createTb(self, tableName, columns, Engine:str='InnoDb', tableComment:str=''): - """ - """ - if self.getTables(tableName): raise ValueError(f'This table [{tableName}] already exist!') - column = ','.join(columns) - - query = f"CREATE TABLE `{tableName}` ({column}) ENGINE = \'{Engine}'" - if tableComment != '': - query += f" COMMENT = '{tableComment}'" - - query+= ';' - self.cursor.execute(query) - - def insert(self, table, columns, values, createTb: bool = False): - - try: - if createTb and not self.getTb(table_name=table): self.createTb(tbName=table, columns=columns, primary_key='id') - if isinstance(columns, (KeysView, list, tuple, ValuesView)): - columns = ','.join([f'`{column}`' for column in columns]) - - if isinstance(values, (KeysView, list, tuple, ValuesView)): - value = [] - for val in values: - if isinstance(val, (KeysView, list, tuple, ValuesView)): - value.append(str(tuple(val))) - elif isinstance(val, str): - value = str(tuple(values)) - break - value = ','.join(value) - elif isinstance(values, str): - value = values - else: - raise ValueError("Invalid values format") - query = f"INSERT INTO `{table}` ({columns}) VALUES {setStr(value)};" - print(query) - self.execute(query) - - except mysql.connector.Error as error: - logging.error("Failed to insert data into MySQL table:", error) - - def fetch(self, table:str, columns:str='*', query: str|list[str]|dict[str, int,str,float]='', limit:int=0, Offset:int=0, fetchAll:bool=True, assc:str='', desc:str='', useIndex:str|None=None, detailed:bool=True, fetchQuery:bool=False)->list[str, float, int]: - """ - fetch() - ------- - - Fetches data from the specified table based on the query. - - Parameters: - - `table` (str): The name of the table to fetch data from. - - `query` (str|list|dict|optional): The SQL query/Search parameter that is to be executed. - - `columns` (str): The columns that needs to be fetched. - - `limit` (int, optioanl): This parameter is to set the number of columns to fetch. - - `offset` (int, optional): The parameter if speciied will get the columns from the limit number of columns to the number specifed in this parameter. eg: from column 5 to 23. This parameter will only be in effect of the limit parameter is use. - - `assc` (str, optional): The columns that are to be fetched in ascending order. - - `desc` (str, optional): The columns that are to be fetched in descending order. - - `fetchAll` (bool, optional): The columns that needs to be fetched. - - Returns: - sqlite3.Row or list of sqlite3.Row: The fetched data. - - Usage Example: - ``` - # assumning cl is the class like. - columns = "column1, column2" - query = "column1 = 'some_value'" - desc = "column2" - data = cl.fetch("my_table", columns, query=query, fetchAll=True, desc=desc) - ``` - """ - - if self.getTb(table) == False: raise ValueError(f'The table({table}) is not present in the database.') - - if isinstance(query, list)and len(query) > 0: - query:str= ' AND '.join([i for i in query if i!='']) - elif isinstance(query, dict)and len(query.keys()) > 0: - query:str= ' AND '.join([f"{k}='{v}'" for k,v in query.items() if v!=None or v!='']) - - query:str= f' WHERE {query}' if query!='' else '' - order:str = '' - if desc != '' or assc != '': - col:list= self.getColumnNames(table) - ord:list = [] - if assc!='': ord.append(f'{assc} ASC' if ',' not in assc else ','.join(f'{i} ASC' for i in assc.split(',') if i in col)) - if desc!='': ord.append(f'{desc} DESC' if ',' not in assc else ','.join(f'{i} DESC' for i in desc.split(',') if i in col)) - order = ' ORDER BY ' + ', '.join(ord) - - if limit > 0: - limit:str = f' LIMIT {str(limit)},{str(Offset)}' if Offset != 0 and Offset > limit else f' LIMIT {str(limit)}' - else: - limit:str = '' - - useidx:str = f' INDEXED BY {useIndex}' if useIndex!=None else '' - sql_query=f"SELECT {columns} FROM {table}{useidx}{query}{order}{limit};" - if fetchQuery: return sql_query - ret = self.execute(sql_query) - try: - if ret is not None: - if detailed==False: - if fetchAll: return ret.fetchall() - else: return ret.fetchone() - else: - data:list = [] - col = [column[0] for column in ret.description] if ret.description else [] - if fetchAll:data = [dict(zip(col, row)) for row in ret.fetchall()] - elif ret.fetchone() is not None: data=dict(zip(col, ret.fetchone())) - logging.info('Fetched Data on quering("%s") successfully', sql_query) - return data - else: return [] - except Exception as e: - logging.error('FETCH Func Error(%e)', sql_query) - print(e) - return [] - - def getColumnNames(self, tableName:str)->list: - """ - getColumnNames() - --------------- - Get the name iof the columns of the given table name. - Parametes: - - tableName str: Name of the table. - - Returns (list): - Retruns the names of the columns of the given table in a list format. - """ - return self.execute(f"SHOW COLUMNS FROM {tableName}") - - def json_insert(self, tableName, data)->None: - """ - JSON_INSERT() - ------------- - - This method is used to insert data into the table using json format data. - Parameters: - - `tableName`(str):This parametere of the method tales the name of the table in which the data is to inserted. - - `data`(list[dict]|dict): This parameter takes the data either in dict format or a list containing dicts if multiple entries are to add. - """ - table_columns = self.getColumnNames(tableName) - columns, queries = [], [] - if isinstance(data, dict): - columns, queries = data.keys(), list(data.values()) - elif isinstance(data, list) and len(data) > 0: - data = equalizer_dict(data) - columns,queries = data[0].keys(), [list(i.values()) for i in data] - else: - raise ValueError('The data type passed is invalid. The data parameter takes a dict or a list of dicts.') - self.insert(tableName, columns,queries) - - def getTb(self, table:str=''): - """ - The method is to get the list of tables form the database. - """ - ret = self.execute("SHOW TABLES") - tbList = [i[0] for i in ret.fetchall()] - if table!='': return table in tbList - return tbList - - def addIndex(self, tableName:str, index_name:str, columns:list): - """ - This method is to add index to the tables. - """ - columns = ','.join(columns) - query = f'`{index_name}` ({columns})' - self.alterTb(tableName, 'ADD UNIQUE', query) - - def alterTb(self, tableName:str, catg:str, query:str): - """ - This method is to used to alter tables in the database. - """ - self.cursor.execute(f"ALTER TABLE `{tableName}` {catg} {query};") - - def cleanTb(self, tableName:str): - """ - This method is to clean the table i.e. it will delete all the data from the table. - """ - if self.getTables(tableName): - self.cursor.execute(f'TRUNCATE TABLE `{tableName}`;') - - def delTb(self, tableName:str): - """This method is to delete the specified table.""" - if self.getTables(tableName): - self.cursor.execute(f'DROP TABLE `{tableName}`;') - - def connect_db(self, dataBase:str, create_db:bool=True): - """ - The task of this mathod is to create new databases in the server. - """ - if self.getDbList(dataBase)==False and create_db: - self.createDb(dataBase) - self.dbConn.database = dataBase - - def createDb(self, dataBase:str): - """ - This method is to crreate a new dataBase. - """ - try: - if self.getDbList(dataBase): return True - return self.execute(f'CREATE DATABASE {dataBase}') - except Exception as e: return e - - def upload_json(self, filePath:str): - """""" - from FileHandler import read, write - data = read(filePath, decode_json=True) - # self.connect_db(data['db_name']) - for tb in data['tables']: - query:list= [] - query.append(f"CREATE TABLE `{tb['table_name']}` ({tuple(tb['column_names'])}) ENGINE = \'InnoDb\';") - columns = [i.split(' ')[0] for i in tb['column_names']] - values = ','.join([str(tuple(i))for i in tb['data']]) - query.append(f'INSERT INTO `{tb['table_name']}` ({columns}) VALUES {values};') - write(f'{data['db_name'].replace('.', '_')}.sql', query, '\n') - - def getDbList(self, present:str=''): - """ - This method gets the list of databases present in the server. - - """ - self.execute("SHOW DATABASES") - dbList = [i[0] for i in self.cursor.fetchall()] - if present != '': return present in dbList - return dbList - - def execute(self, query:str): - """This method is to execute the mysql queries.""" - try: - d = self.cursor.execute(query) - self.dbConn.commit() - logging.success(f'Exceution success ({query})') - return d - except Exception as e: - logging.error(f'Exceution error ({query}) -> {e}') - - def delDb(self, dataBase:str): - """This method is to delete dataBases.""" - if self.getDbList(dataBase): - self.execute(f'DROP DATABASE {dataBase}') - else: - raise ValueError('Database does not exist!') - - def close_connection(self): - """This method is to close the connection.""" - if (self.dbConn.is_connected()): - self.cursor.purge() - self.cursor.close() - self.dbConn.close() \ No newline at end of file diff --git a/DbHandler/__init__.py b/DbHandler/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/FileHandler.py b/FileHandler.py index 1301e43..604c8c9 100644 --- a/FileHandler.py +++ b/FileHandler.py @@ -1,8 +1,7 @@ import json, csv, os from openpyxl import Workbook, load_workbook from PyPDF2 import PdfReader -from DataHandlers import remove_empty_strings, dlist_dict, list_dlist, decode_json, dict_lister - +from DataHandlers import remove_empty_strings, dlist_dict, list_dlist, decode_json def getFiles(file_path:str='', catg:int=0, extention=None, full_path:bool=False): @@ -22,7 +21,7 @@ def getFiles(file_path:str='', catg:int=0, extention=None, full_path:bool=False) files = os.listdir(file_path) if full_path or catg==1 or extention!=None: - files = [f'{file_path}/{file}' for file in files] + files = [os.path.join(file_path, file) for file in files] if catg == 1 or extention != None: if extention!=None: @@ -125,13 +124,10 @@ def get_file_info(self, file:str)->dict: } return data -def fileExists(file:str)->bool: +def fileExists(file:str): """Checks if a file exists or not""" return True if os.path.exists(file) and os.path.isfile(file) else False -def delFiles(filePath:str)->None: - if fileExists(filePath): os.remove(filePath) - def write(file_name: str, data, separator='', mode:str='a', write_bytes:bool=False, emptyPervious:bool=False) -> None: """ Appends data to a file. @@ -145,17 +141,18 @@ def write(file_name: str, data, separator='', mode:str='a', write_bytes:bool=Fal """ if emptyPervious: - mode:str= 'w' - if write_bytes: - mode='wb' if emptyPervious else 'ab' + mode = 'w' + if write_bytes: mode=mode+'b' if isinstance(data,(list, tuple, set)): - for d in data: write(file_name, d, separator) + [write(file_name, d, separator) for d in data] else: with open(file_name, mode, encoding='utf-8', errors='ignore') as file: data_type = type(data) - if data_type == str: file.write(data) - elif data_type == dict: json.dump(data, file) + if data_type == str: + file.write(data) + elif data_type == dict: + json.dump(data, file) else: TypeError('The data type provided is not supported.') file.write(separator) @@ -172,12 +169,23 @@ def read(file_name: str, splitter=None, encode='utf-8', error='ignore', mode:str :return: The read data. """ if read_bytes: - mode, encode, error = 'rb',None, None + mode = 'rb' + encode, error = None, None with open(file_name, mode, encoding=encode, errors=error) as file: data = file.read() - if splitter is not None: return [(json.loads(ret) if decode_json else ret) for ret in remove_empty_strings(data.split(splitter))] - else: return json.loads(data) if decode_json else data + retdata = [] + if splitter is not None: + for ret in remove_empty_strings(data.split(splitter)): + if decode_json: + ret = json.loads(ret) + retdata.append(ret) + return retdata + else: + if decode_json: + return json.loads(data) + else: + return data def write_csv(file_name: str, data, header=None): """ @@ -212,39 +220,19 @@ def read_csv(file_name: str, header:bool=False): return dlist_dict(list_dlist(data[1:], data[0])) return data -# def write_excel(filename, data, mode:str='write', sheetname='Sheet 1', column:int=1, row:int=1): - # if mode not in ['append', 'write']: - # raise ValueError("Mode should be either 'append' or 'write'") - - # if mode == 'wr' or not os.path.exists(filename): - # workbook = Workbook() - # worksheet = workbook.active - # worksheet.title = sheetname - # else: - # workbook = load_workbook(filename) - # worksheet = workbook.active - - # if isinstance(data, dict): - # for k,v in data.items(): - # write_excel(filename, data=v, mode='append', sheetname=k) - # else: - # if isinstance(data, list) and isinstance(data[0], dict): - # data_reform = dict_lister(data) - # row:int = 1 - # col:int = 1 - # for i, he in enumerate(list(data_reform.keys())): - # worksheet.cell(row=row, column=i+1, value=he) - # row+=1 - # for _,v in data_reform.items(): - # for i,val in enumerate(v): - # worksheet.cell(row=i+row, column=col, value=val) - # col+=1 - # elif isinstance(data, list) and isinstance(data[0], list): - # for i, row in enumerate(data): - # for j, value in enumerate(row): - # worksheet.cell(row=i+1, column=j+1, value=value) - - # workbook.save(filename) +def write_excel(filename, data, sheetname='Sheet 1'): + workbook = Workbook() + sheet = workbook.active + sheet.title = sheetname + if isinstance(data, list) and isinstance(data[0], dict): + pass + elif isinstance(data, list) and isinstance(data[0], list): + for i, row in enumerate(data): + for j, value in enumerate(row): + sheet.cell(row=i+1, column=j+1, value=value) + elif isinstance(data, dict): + pass + workbook.save(filename) def read_excel(file_name: str, sheet_name=None, organize:bool=False): """ diff --git a/HtmlScraper.py b/HtmlScraper.py index 32e89d0..39b76a2 100644 --- a/HtmlScraper.py +++ b/HtmlScraper.py @@ -1,182 +1,182 @@ -from Requester import Requester -from bs4 import BeautifulSoup -from FileHandler import write, read -from DataHandlers import get_unique - -class HtmlScraper: - - def __init__(self, url:str, setSessions:bool=False, set_header:bool=True, set_agent:bool=True, set_proxy:bool=False)->None: - """ - HtmlScraper - =========== - - This is a class that is to be used to scrape a website. - """ - self.url = url - self.setSessions, self.sessions = setSessions, None - self.req = Requester(set_header=set_header, set_agent=set_agent, set_proxy=set_proxy, agent_file='F:/Code Works/Python_works/storage/others/user-agent.txt') # proxy_file='F:/Code Works/Python_works/storage/others/proxies.txt') - self.souped = None - - def _rectiftyPathway(self, pathway): - - if isinstance(pathway, list): - pathway = [self._rectiftyPathway(p) for p in pathway] - elif isinstance(pathway, dict): - r = pathway.keys() - if 'tag' in r: - if 'attr' not in r: - pathway['attr'] = {} - if 'type' not in r: - pathway['type'] = 'find' - else: - pathway = {k: self._rectiftyPathway(v) for k,v in pathway.items()} - return pathway - - def _request(self, method:str='get', params:dict={}, ref:str='', response_code:int=200): - reqVals = {'url': self.url, 'method': method,'params': params, 'ref': ref, 'response_code': response_code} - req = None - if self.setSessions: - req, self.sessions = self.req.requestSessions(sessions=self.sessions, **reqVals) - else: - req = self.req.request(**reqVals) - return req - - def _souper(self, data, parser:str='html.parser'): - """ - _souper() - --------- - - Converts a html document data into BeautifulSoup class value. - """ - self.souped = BeautifulSoup(data, parser) - return self.souped - - def _getAtr(self, data, ty): - if isinstance(data, list): - return [self._getAtr(t, ty) for t in data if t is not None] - else: - r = None - if ty=='' or ty=='': - r = data.getText(strip=(True if ty=='' else False)) - else: - r = data.get(ty) - return r - - def _parser(self, selectorType:str='find', tagName='', attribute:dict={}, data=None)->(list|str|None): - """ - _parser() - --------- - This method is responsible for fetching the target element in the document. - - Parameters: - - `selectorType` str: The type of method that is to be used for fetching an element(s).Its values are - - find: Finds a single element, the first element that matches the values (tagName & attribute). Also the default value. - - find_all: Finds all the element of the same tag and atribute value. - - select_one: Similar to find, the tagName used would be the JS query selector value. Selecting this value returns a single value. - - select: Similar to find_all, the tagName used would be the JS query selector value. Selecting this value returns a list of value. - - `tagName` str: This parameter determines where to lookand what to fetch. Depending of on the `selectorType`, the value can be a tagName(for `find` & `find_all`) or a JS querySelctor value (for `select` or `select_one`) - - `attribbute` dict: This acts as a supporter for finding the target tag value. - - `data`: This parameter is to pass the html data where to look. Default is `None` which means, the page that will be parsed will be the page got during the requesting of the page. - - Returns: - - NoneType|list|str: Depending on the value passsed in `selectorType` parameter, the data type passed can be a list, str or a None type value. - - `select_one` or `find`: Return str - - `select` or `find_all`: Return list - - If no data found: Return None - """ - if data==None: - data=self.souped - try: - if selectorType == 'select': - k = data.select(tagName, attr=attribute) - elif selectorType == 'select_one': - k = data.select_one(tagName, attr=attribute) - elif selectorType == 'find': - k = data.find(tagName, attr=attribute) - elif selectorType == 'findall' or selectorType =='find_all': - k = data.find_all(tagName, attr=attribute) - return k - except: - return None - - # basic purpose - def storePage(self, fileName:str, data=None, seperator:str='\n', prevEmpty:bool=True)->None: - """ - storePage() - ----------- - This method is to store the page or the data in a file. - - Parameter: - - fileName str: Name of the file. - - data any: Takes the data that is to be inserted in the file. Default is `None`, which means the data stored will be the html data fetched during the request processes. - - seperator str: THis paramerter specifies, how the data points will be seperated in the file. Default is `\n` (a line break). - - prevEmpty bool: This parameter specifies if the existing data in the file should remain or be deleted. Default is `True`. Values:- - - `True`: The file will be emptied before inserting new data. - - `False`: The new data will be appended into the file with existing data. - """ - write(file_name=fileName, data=(self.souped if data==None else data), separator=seperator, emptyPervious=prevEmpty) - - # User use functions - def getAllUrls(self, data=None)->list[str]: - """ - getAllUrls() - ------------ - This method is fo getting all the urls in the parsed page - """ - return get_unique([i.get('href') for i in self._parser(selectorType='find_all',tagName='a', data=data)]) - - def getAllImages(self, data=None): - """ - getAllImages() - -------------- - Returns all the images in the page. - """ - imgs = {'tag': 'img', 'attr': {}, 'type': 'select', 'inner':{'imgLnk': 'src', 'alt':'alt'}} - return self.jsonParser(pathway=imgs, data=data) - - def getPageMeta(self, data=None): - """ - getPageMeta() - ------------- - Fetches the meta data of the page. - """ - pathway = {'title': {'tag':'title', 'get': ''}} - return self.jsonParser(pathway, data) - - def jsonParser(self, pathway:dict, data=None)->dict|None|list: - """ - jsonParser() - ------------ - This method is responsible for parsing the websitein the given structure. - """ - if data==None: - data = self._souper(self._request()) if self.souped==None else self.souped - - pathway = self._rectiftyPathway(pathway) - try: - if isinstance(pathway, str): - return self._parser(selectorType='select', tag=pathway, data=data) - elif isinstance(pathway, dict): - ret:dict = {} - if len(pathway) == 0: return None - if 'tag' in pathway.keys(): - k = self._parser(selectorType=pathway['type'], tagName=pathway['tag'], attribute=pathway['attr'], data=data) - if k is not None: - if 'get' in pathway.keys() and pathway['get'] != '' and pathway['get'] is not None: - ret['get'] = self._getAtr(ty=pathway['get'], data=k) - - if 'inner' in pathway.keys() and pathway['inner']!={}: - ret['inner'] = [self.jsonParser(data=n,pathway=pathway['inner']) for n in k] if isinstance(k, list) else self.jsonParser(data=k, pathway=pathway['inner']) - elif 'get' in ret.keys(): - return ret['get'] - else: return k - else: - for k,v in pathway.items(): - ret[k] = self._getAtr(ty=v, data=data) if isinstance(v, str) else self.jsonParser(pathway=v,data=data) - - return ret - elif isinstance(pathway, list): - return [self.jsonParser(pathway=i,data=data) for i in pathway] - except: - return None - +from Requester import Requester +from bs4 import BeautifulSoup +from FileHandler import write, read +from DataHandlers import get_unique + +class HtmlScraper: + + def __init__(self, url:str, setSessions:bool=False, set_header:bool=True, set_agent:bool=True, set_proxy:bool=False)->None: + """ + HtmlScraper + =========== + + This is a class that is to be used to scrape a website. + """ + self.url = url + self.setSessions, self.sessions = setSessions, None + self.req = Requester(set_header=set_header, set_agent=set_agent, set_proxy=set_proxy, agent_file='F:/Code Works/Python_works/storage/others/user-agent.txt') # proxy_file='F:/Code Works/Python_works/storage/others/proxies.txt') + self.souped = None + + def _rectiftyPathway(self, pathway): + + if isinstance(pathway, list): + pathway = [self._rectiftyPathway(p) for p in pathway] + elif isinstance(pathway, dict): + r = pathway.keys() + if 'tag' in r: + if 'attr' not in r: + pathway['attr'] = {} + if 'type' not in r: + pathway['type'] = 'find' + else: + pathway = {k: self._rectiftyPathway(v) for k,v in pathway.items()} + return pathway + + def _request(self, method:str='get', params:dict={}, ref:str='', response_code:int=200): + reqVals = {'url': self.url, 'method': method,'params': params, 'ref': ref, 'response_code': response_code} + req = None + if self.setSessions: + req, self.sessions = self.req.requestSessions(sessions=self.sessions, **reqVals) + else: + req = self.req.request(**reqVals) + return req + + def _souper(self, data, parser:str='html.parser'): + """ + _souper() + --------- + + Converts a html document data into BeautifulSoup class value. + """ + self.souped = BeautifulSoup(data, parser) + return self.souped + + def _getAtr(self, data, ty): + if isinstance(data, list): + return [self._getAtr(t, ty) for t in data if t is not None] + else: + r = None + if ty=='' or ty=='': + r = data.getText(strip=(True if ty=='' else False)) + else: + r = data.get(ty) + return r + + def _parser(self, selectorType:str='find', tagName='', attribute:dict={}, data=None)->(list|str|None): + """ + _parser() + --------- + This method is responsible for fetching the target element in the document. + + Parameters: + - `selectorType` str: The type of method that is to be used for fetching an element(s).Its values are + - find: Finds a single element, the first element that matches the values (tagName & attribute). Also the default value. + - find_all: Finds all the element of the same tag and atribute value. + - select_one: Similar to find, the tagName used would be the JS query selector value. Selecting this value returns a single value. + - select: Similar to find_all, the tagName used would be the JS query selector value. Selecting this value returns a list of value. + - `tagName` str: This parameter determines where to lookand what to fetch. Depending of on the `selectorType`, the value can be a tagName(for `find` & `find_all`) or a JS querySelctor value (for `select` or `select_one`) + - `attribbute` dict: This acts as a supporter for finding the target tag value. + - `data`: This parameter is to pass the html data where to look. Default is `None` which means, the page that will be parsed will be the page got during the requesting of the page. + + Returns: + - NoneType|list|str: Depending on the value passsed in `selectorType` parameter, the data type passed can be a list, str or a None type value. + - `select_one` or `find`: Return str + - `select` or `find_all`: Return list + - If no data found: Return None + """ + if data==None: + data=self.souped + try: + if selectorType == 'select': + k = data.select(tagName, attr=attribute) + elif selectorType == 'select_one': + k = data.select_one(tagName, attr=attribute) + elif selectorType == 'find': + k = data.find(tagName, attr=attribute) + elif selectorType == 'findall' or selectorType =='find_all': + k = data.find_all(tagName, attr=attribute) + return k + except: + return None + + # basic purpose + def storePage(self, fileName:str, data=None, seperator:str='\n', prevEmpty:bool=True)->None: + """ + storePage() + ----------- + This method is to store the page or the data in a file. + + Parameter: + - fileName str: Name of the file. + - data any: Takes the data that is to be inserted in the file. Default is `None`, which means the data stored will be the html data fetched during the request processes. + - seperator str: THis paramerter specifies, how the data points will be seperated in the file. Default is `\n` (a line break). + - prevEmpty bool: This parameter specifies if the existing data in the file should remain or be deleted. Default is `True`. Values:- + - `True`: The file will be emptied before inserting new data. + - `False`: The new data will be appended into the file with existing data. + """ + write(file_name=fileName, data=(self.souped if data==None else data), separator=seperator, emptyPervious=prevEmpty) + + # User use functions + def getAllUrls(self, data=None)->list[str]: + """ + getAllUrls() + ------------ + This method is fo getting all the urls in the parsed page + """ + return get_unique([i.get('href') for i in self._parser(selectorType='find_all',tagName='a', data=data)]) + + def getAllImages(self, data=None): + """ + getAllImages() + -------------- + Returns all the images in the page. + """ + imgs = {'tag': 'img', 'attr': {}, 'type': 'select', 'inner':{'imgLnk': 'src', 'alt':'alt'}} + return self.jsonParser(pathway=imgs, data=data) + + def getPageMeta(self, data=None): + """ + getPageMeta() + ------------- + Fetches the meta data of the page. + """ + pathway = {'title': {'tag':'title', 'get': ''}} + return self.jsonParser(pathway, data) + + def jsonParser(self, pathway:dict, data=None)->dict|None|list: + """ + jsonParser() + ------------ + This method is responsible for parsing the websitein the given structure. + """ + if data==None: + data = self._souper(self._request()) if self.souped==None else self.souped + + pathway = self._rectiftyPathway(pathway) + try: + if isinstance(pathway, str): + return self._parser(selectorType='select', tag=pathway, data=data) + elif isinstance(pathway, dict): + ret:dict = {} + if len(pathway) == 0: return None + if 'tag' in pathway.keys(): + k = self._parser(selectorType=pathway['type'], tagName=pathway['tag'], attribute=pathway['attr'], data=data) + if k is not None: + if 'get' in pathway.keys() and pathway['get'] != '' and pathway['get'] is not None: + ret['get'] = self._getAtr(ty=pathway['get'], data=k) + + if 'inner' in pathway.keys() and pathway['inner']!={}: + ret['inner'] = [self.jsonParser(data=n,pathway=pathway['inner']) for n in k] if isinstance(k, list) else self.jsonParser(data=k, pathway=pathway['inner']) + elif 'get' in ret.keys(): + return ret['get'] + else: return k + else: + for k,v in pathway.items(): + ret[k] = self._getAtr(ty=v, data=data) if isinstance(v, str) else self.jsonParser(pathway=v,data=data) + + return ret + elif isinstance(pathway, list): + return [self.jsonParser(pathway=i,data=data) for i in pathway] + except: + return None + diff --git a/LICENSE b/LICENSE index 55a8635..42f9bc5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,9 @@ MIT License -Copyright (c) 2024 Ranit Saha +Copyright 2024 Coderooz -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the β€œSoftware”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +THE SOFTWARE IS PROVIDED β€œAS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/Readme.md b/Readme.md index 0ebbc01..67b6068 100644 --- a/Readme.md +++ b/Readme.md @@ -1,315 +1,479 @@ -# Personal EveryDay Usage Functions +# Custom Functions -## Project Author Details: -- Name: Ranit Saha -- Code Name: Codezees -- Guthub Profile: https://github.com/Codezees +[![Python Version](https://img.shields.io/badge/python-3.12%2B-blue.svg)](https://www.python.org/downloads/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.txt) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg)](CONTRIBUTING.md) +[![Status](https://img.shields.io/badge/status-alpha-orange.svg)](CHANGELOG.md) +A collection of pre-made Python utilities for everyday tasks. This package provides ready-to-use classes for data handling, database operations, file I/O, and HTTP requests β€” helping you build projects faster and more effectively. -## Contents: +## Features -1. [Project Info](#project_info) -2. [Project Files](#project_files) -3. [Files Description](#project_file_description) +- **DataHandler**: Timestamp formatting, data normalization, and common data manipulation utilities +- **DbHandler**: Full-featured SQLite database management with CRUD operations, table schema modifications, and index management +- **FileHandler**: Comprehensive file operations including read/write, CSV/Excel/PDF handling, and file system utilities +- **Requester**: Simplified HTTP request handling with common patterns and utilities -## Project Info -

This is just a personal project where I make classes and fuctions that will help me do certain task more effectiely rather that re-codig them again and again.

+## Installation -## Project Files -Project Files are:- -- [DataHandler](#dataHandler_file) -- [DbHandler](#dbsqlithandler_file) -- [Filehandler](#fileHandler_file) -- [Requester](#requester_file) +### From PyPI (Recommended) -## Files Description -File Descriptions +```bash +pip install custom_functions +``` -### DataHandler -

The file DataHandler encases a class called DataHandler(). This class is used to menial tasks that usually require rewriting of a lot of code. +### From Source -This class is used for handleling data related tasks -

+```bash +# Clone the repository +git clone https://github.com/coderooz/My_simple_functions.git +cd My_simple_functions + +# Install in development mode +pip install -e . + +# Or install with development dependencies +pip install -e ".[dev]" +``` + +### Requirements + +- Python 3.12 or higher +- Dependencies (installed automatically): + - `pytz>=2023.3.0` + - `requests>=2.31.0` + - `db-sqlite3>=0.0.1` + - `pandas>=2.1.3` + - `mysql-connector-python>=2.2.9` + - `openpyxl>=3.1.2` + - `PyPDF2>=3.0.0` + +## Quick Start + +```python +# Import the handlers +from DataHandlers import DataHandler +from DbHandler import DbHandler +from FileHandler import FileHandler +from Requester import Requester + +# Use DataHandler for timestamp formatting +timestamp = DataHandler.timestamp() +print(timestamp) # Output: 2024-01-15 10:30:45 + +# Use DbHandler for database operations +db = DbHandler('my_database.db') +db.createTb('users', ['name TEXT', 'email TEXT']) +db.insert('users', ['John Doe', 'john@example.com']) + +# Use FileHandler for file operations +FileHandler.write('output.txt', 'Hello, World!') +content = FileHandler.read('output.txt') + +# Use Requester for HTTP requests +# requester = Requester() +# response = requester.get('https://api.example.com/data') +``` + +## Detailed Usage + +### DataHandler + +The `DataHandler` class provides utilities for common data manipulation tasks. + +#### `timestamp(given_time=None, format="%Y-%m-%d %H:%M:%S", time_zone=None, normalize='sec')` + +Returns the date and time in the specified format. + +**Parameters:** +- `given_time`: Optional datetime object or timestamp (default: current time) +- `format`: strftime format string (default: `"%Y-%m-%d %H:%M:%S"`) +- `time_zone`: Optional timezone string (e.g., `"Asia/Kolkata"`) +- `normalize`: Normalization level β€” `'sec'`, `'min'`, or `'hour'` (default: `'sec'`) + +**Returns:** `str` β€” Formatted timestamp string + +**Example:** +```python +from DataHandlers import DataHandler + +# Current timestamp +print(DataHandler.timestamp()) +# Output: 2024-01-15 10:30:45 + +# Custom format +print(DataHandler.timestamp(format="%Y-%m-%d")) +# Output: 2024-01-15 + +# With timezone +print(DataHandler.timestamp(time_zone="US/Pacific")) +# Output: 2024-01-14 21:00:45 +``` + +--- + +### DbHandler + +The `DbHandler` class provides a comprehensive interface for SQLite database operations. + +#### Initialization + +```python +from DbHandler import DbHandler + +# Creates new database or connects to existing one +db = DbHandler('my_database.db') +``` + +#### Core Methods + +##### `createTb(table_name, columns, insertData=None, addId=False, idKey='id')` + +Creates a new table with optional data insertion. + +**Parameters:** +- `table_name`: Name of the table +- `columns`: List of column definitions (e.g., `['col1 TEXT', 'col2 INT']`) +- `insertData`: Optional data to insert immediately after creation +- `addId`: Whether to add an auto-incrementing primary key (default: `False`) +- `idKey`: Name of the primary key column (default: `'id'`) + +**Example:** +```python +# Create table with columns +db.createTb('users', ['name TEXT', 'email TEXT', 'age INT']) + +# Create with auto ID and initial data +db.createTb('users', ['name TEXT', 'email TEXT'], + insertData=['John', 'john@example.com'], + addId=True) +``` + +##### `insert(table_name, data)` + +Inserts a row into the specified table. + +**Example:** +```python +db.insert('users', ['Alice', 'alice@example.com', 25]) +``` + +##### `fetch(table_name, columns='*', query='')` + +Fetches data from a table with optional filtering. + +**Example:** +```python +# Fetch all rows +all_users = db.fetch('users') + +# Fetch specific columns with condition +result = db.fetch('users', columns='name, email', query="age > 18") +``` + +##### `getCount(table_name, query='')` + +Counts rows in a table, optionally with a filter. + +**Example:** +```python +total = db.getCount('users') +adults = db.getCount('users', query="age >= 18") +``` + +##### `update(table_name, data, query)` + +Updates rows matching the query. + +**Example:** +```python +db.update('users', {'age': 26}, query="name='Alice'") +``` + +##### `execute(query, params=None)` + +Executes raw SQL queries. + +**Example:** +```python +db.execute("DROP TABLE IF EXISTS temp_table") +``` + +#### Additional Methods + +| Method | Description | +|--------|-------------| +| `delTb(table_name)` | Delete a table | +| `renameTb(old_name, new_name)` | Rename a table | +| `getTb()` | List all tables | +| `getTbData(table_name)` | Get all data from a table | +| `alterTb(table_name, operation)` | Alter table structure | +| `getColumnNames(table_name)` | Get column names | +| `get_table_info(table_name)` | Get detailed table info | +| `modifyColumns(table_name, columns)` | Modify column definitions | +| `checkIndex(table_name, index_name)` | Check if index exists | +| `getIndexes(table_name)` | Get all indexes | +| `addIndex(table_name, columns, unique=False)` | Add an index | +| `delIndex(index_name)` | Delete an index | +| `cleanTb(table_name)` | Delete all rows from a table | +| `addColumn(table_name, column_def)` | Add a new column | +| `renameColumn(table_name, old_name, new_name)` | Rename a column | +| `removeColumn(table_name, column_name)` | Remove a column | +| `close_connection()` | Close database connection | + +--- + +### FileHandler + +The `FileHandler` class provides utilities for file system operations and file format handling. + +#### Initialization + +```python +from FileHandler import FileHandler + +# Static class β€” no initialization needed +# Use directly: FileHandler.method_name() +``` + +#### Core Methods + +##### `getFiles(file_path, catg=0)` + +Gets list of files and/or folders in a directory. + +**Parameters:** +- `file_path`: Path to the directory +- `catg`: Category filter β€” `0` for both, `1` for files only, `2` for directories only + +**Returns:** `list` of file/directory names + +**Example:** +```python +# Get both files and folders +items = FileHandler.getFiles('./my_folder') -Functions of the class:- -- timestamp: Returns the date and time in the specified format. - +# Get only files +files = FileHandler.getFiles('./my_folder', catg=1) + +# Get only directories +dirs = FileHandler.getFiles('./my_folder', catg=2) +``` + +##### `get_only_filename(file_path)` + +Extracts just the filename from a path. + +##### `getFileCatg(file_path)` + +Gets the file category/type based on extension. + +##### `splitFileName(file_path)` + +Splits filename into name and extension. + +##### `getExtention(file_path)` + +Gets the file extension. + +##### `read(file_name)` + +Reads and returns file contents. + +##### `write(file_name, data, separator='')` + +Appends data to a file. + +**Parameters:** +- `file_name`: Path to the file +- `data`: Content to write +- `separator`: Optional separator to append after data + +**Example:** +```python +FileHandler.write('output.txt', 'Hello, World!', separator='\n') +``` + +##### `write_over(file_name, data, separator='')` + +Overwrites file contents with new data. + +##### `read_csv(file_name, **kwargs)` + +Reads a CSV file and returns a pandas DataFrame. + +##### `write_csv(file_name, data, **kwargs)` + +Writes data to a CSV file. + +##### `read_excel(file_name, **kwargs)` + +Reads an Excel file and returns a pandas DataFrame. + +##### `write_excel(file_name, data, **kwargs)` + +Writes data to an Excel file. + +##### `read_pdf(file_name)` + +Extracts and returns text content from a PDF file. + +**Example:** +```python +text = FileHandler.read_pdf('document.pdf') +print(text) +``` + +--- + +### Requester + +The `Requester` class provides simplified HTTP request handling. + +```python +from Requester import Requester + +# Initialize +requester = Requester() + +# GET request +response = requester.get('https://api.example.com/data') + +# POST request +response = requester.post('https://api.example.com/data', json={'key': 'value'}) + +# With custom headers +response = requester.get('https://api.example.com/data', + headers={'Authorization': 'Bearer token'}) +``` + +## Project Structure + +``` +custom_functions/ +β”œβ”€β”€ DataHandlers.py # Data manipulation utilities +β”œβ”€β”€ DbHandler.py # SQLite database operations +β”œβ”€β”€ FileHandler.py # File I/O and format handling +β”œβ”€β”€ Requester.py # HTTP request utilities +β”œβ”€β”€ __init__.py # Package initialization +β”œβ”€β”€ setup.py # Package distribution setup +β”œβ”€β”€ pyproject.toml # Modern Python project configuration +β”œβ”€β”€ LICENSE.txt # MIT License +β”œβ”€β”€ README.md # This file +β”œβ”€β”€ CHANGELOG.md # Version history +β”œβ”€β”€ CONTRIBUTING.md # Contribution guidelines +β”œβ”€β”€ CODE_OF_CONDUCT.md # Community guidelines +β”œβ”€β”€ SECURITY.md # Security policy +β”œβ”€β”€ .gitignore # Git ignore rules +β”œβ”€β”€ .editorconfig # Editor configuration +└── .github/ # GitHub-specific files + β”œβ”€β”€ ISSUE_TEMPLATE/ # Issue templates + β”œβ”€β”€ workflows/ # CI/CD workflows + β”œβ”€β”€ CODEOWNERS # Code ownership + β”œβ”€β”€ dependabot.yml # Dependency updates + β”œβ”€β”€ labels.yml # Issue labels + β”œβ”€β”€ PULL_REQUEST_TEMPLATE.md + └── FUNDING.yml # Sponsorship information +``` + +## Development + +### Setup Development Environment + +```bash +# Clone and set up virtual environment +git clone https://github.com/coderooz/My_simple_functions.git +cd My_simple_functions +python -m venv venv +source venv/bin/activate # Windows: venv\Scripts\activate + +# Install with dev dependencies +pip install -e ".[dev]" ``` -data = timestamp(given_time, format= "%Y-%m-%d %H:%M:%S", time_zone=None, normalize:str='sec') -print(data) -#output: + +### Running Tests + +```bash +# Run all tests +pytest + +# Run with coverage +pytest --cov=custom_functions --cov-report=html + +# Run specific test file +pytest tests/test_datahandler.py -v +``` + +### Code Quality + +```bash +# Format code +black . + +# Sort imports +isort . + +# Lint +flake8 . + +# Run all checks +black . && isort . && flake8 . ``` -### DbSqliteHandler -

The file DbSqliteHandler encases a class called DbSqliteHandler. This class is used creating and managing SQLITE3 databases. - -- Initializaton of database: -When initalizing the database, on emust provide the desired database name. If the database existes, the class will just connect it else if the the database doesnot exist, then the database will be created and connectionn will be established. - ```` - import DbSqliteHandler - dbConn = DbSqliteHandler('db_name.db') - ```` -- create(): -If the database is created newly, then the database will require a table to function properly, or else the db will just remain as file in the system. -So to create a table we need to use the class method called createTb(). - - Parameter: - - Usage: - ``` - tb_name = 'table_1' - columns = ['col1', 'col2'] - ## or you can specify the datatype like this. - columns = ['col1 TEXT', 'col2 INT'] - dbConn.createTb(tb_name, columns) - ``` - The method also comes with the feature of adding data along side creating the table raher than writing a special code for it and also comes with the feature to ```primary key``` while creating the table. - - ``` - col = ['col1', 'col2'] - data = ['data1', 'data2'] - dbConn.createTb(tbName, columns=col, insertData=data, addId=False, idKey='id') - ``` -- insert: - - Parameter: - - Usage: - ``` - - ``` -- json_insert: - - Parameter: - - Usage: - ``` - - ``` -- fetch: - - Parameter: - - Usage: - ``` - - ``` -- getCount: This method counts the number of rows in the specified table. This method also allows or count with special queries like `col1='tofu'`. - - Parameter: - - `table_name` : Takes the name of table. - - `query` : Takes the search query. Default is `''` meaning, there is no search parameter and will count all the rows in the table. - - Returns: `int` - - Usage: - ``` - query = "col1='tofu'" - num = dbconn.getCount(table_name, query) - print(num) ##prints out the number of row with col1 value as tofu. - ``` -- update: - - Parameter: - - Usage: - ``` - - ``` -- delTb: - - Parameter: - - Usage: - ``` - - ``` -- renameTb: - - Parameter: - - Usage: - ``` - - ``` -- getTb: - - Parameter: - - Usage: - ``` - - ``` -- getTbData: - - Parameter: - - Usage: - ``` - - ``` -- execute: - - Parameter: - - Usage: - ``` - - ``` -- alterTb: - - Parameter: - - Usage: - ``` - - ``` -- getColumnNames: - - Parameter: - - Usage: - ``` - - ``` -- get_table_info: - - Parameter: - - Usage: - ``` - - ``` -- modifyColumns: - - Parameter: - - Usage: - ``` - - ``` -- checkIndex: - - Parameter: - - Usage: - ``` - - ``` -- getIndexes: - - Parameter: - - Usage: - ``` - - ``` -- addIndex: - - Parameter: - - Usage: - ``` - - ``` -- delIndex: - - Parameter: - - Usage: - ``` - - ``` -- cleanTb: - - Parameter: - - Usage: - ``` - - ``` -- addColumn: - - Parameter: - - Usage: - ``` - - ``` -- renameColumn: - - Parameter: - - Usage: - ``` - - ``` -- removeColumn: Removes a column from the specified table. - - Parameter: - - Usage: - ``` - - ``` -- close_connection:This method for to close all the connections made to the db and also end the session. - - - -### Filehandler -

The file Filehandler encases a class called Filehandler. This class is used for writing & reading file related tasks. - -- Initialization: The initiallization is a simple process. To use it , just add `FileHandler.func_name()` and it will work. Examples are below. -- Class methods: The class method are as follows: - - `getFiles()`: Gets the list for files and folder in the specified folder. - - Parameter: - - `file_path`: Takes the path of the folder that needs to be looked into. - - `catg` : This parameter takes either 1 or 2, where 1 means to only look for files whereas 2 means to lik for directories. Default is 0. Which means both. - - Return: `list` - - Usage: - ``` - folder_path = './test_folder' #folder present in the current directory. - fi = getFiles(folder_path) # for both files and folder. - print(fi) #output: [file1.txt, dir1, ....] - ``` - - `get_only_filename()`: - - `getFileCatg()`: - - `splitFileName()`: - - `getExtention()`: - - `read()`: - - `write()`: This function is to write files and add contents to it. - - Parameter: - - `file_name` : The name of the file. - - `data`: The data to be written to the file. - - `separator`: Optional separator to append after the data. - - Usage: - ``` - data = 'This is Hello World File.' #file contents - fileName = 'test.txt' # the file name - FileHandler.write() - ``` - - `write_over()`: - - Parameter: - - `file_name` : The name of the file. - - `data`: The data to be written to the file. - - `separator`: Optional separator to append after the data. - - Usage: - ``` - data = 'This is Hello World File.' #file contents - fileName = 'test.txt' # the file name - FileHandler.write() - ``` - - `read_csv()`: - - Parameter: - - `file_name` : The name of the file. - - `data`: The data to be written to the file. - - `separator`: Optional separator to append after the data. - - Usage: - ``` - data = 'This is Hello World File.' #file contents - fileName = 'test.txt' # the file name - FileHandler.write() - ``` - - `write_csv()`: - - Parameter: - - `file_name` : The name of the file. - - `data`: The data to be written to the file. - - `separator`: Optional separator to append after the data. - - Usage: - ``` - data = 'This is Hello World File.' #file contents - fileName = 'test.txt' # the file name - FileHandler.write() - ``` - - `read_excel()`: - - Parameter: - - `file_name` : The name of the file. - - `data`: The data to be written to the file. - - `separator`: Optional separator to append after the data. - - Usage: - ``` - data = 'This is Hello World File.' #file contents - fileName = 'test.txt' # the file name - FileHandler.write() - ``` - - `write_excel()`: - - Parameter: - - `file_name` : The name of the file. - - `data`: The data to be written to the file. - - `separator`: Optional separator to append after the data. - - Usage: - ``` - data = 'This is Hello World File.' #file contents - fileName = 'test.txt' # the file name - FileHandler.write() - ``` - - `read_pdf()`: This function reads and returns the text values of the pdf. - - Parameter: - - `file_name` : The name of the file. - - Usage: - ``` - fileName = 'test.pdf' # the file name - FileHandler.read_pdf() - ``` - - -### Requester -

The file Requester encases a class called Requester. This class is used to request related tasks. - - -## +## Contributing + +We welcome contributions of all kinds! Please read our [Contributing Guide](CONTRIBUTING.md) for details on: + +- How to set up your development environment +- Our coding standards and conventions +- How to submit pull requests +- How to report bugs or request features + +### Quick Start for Contributors + +1. Fork the repository +2. Create a feature branch (`git checkout -b feature/amazing-feature`) +3. Make your changes +4. Run tests and linting +5. Commit your changes (`git commit -m 'feat: add amazing feature'`) +6. Push to the branch (`git push origin feature/amazing-feature`) +7. Open a Pull Request + +Please read our [Code of Conduct](CODE_OF_CONDUCT.md) before contributing. + +## Versioning + +We use [Semantic Versioning](https://semver.org/) (SemVer) for versioning. See the [CHANGELOG.md](CHANGELOG.md) for a list of available versions and the changes in each release. + +## License + +This project is licensed under the MIT License β€” see the [LICENSE.txt](LICENSE.txt) file for details. +## Author +**Ranit Saha** (Coderooz) + +- GitHub: [@Coderooz](https://github.com/coderooz) +- Website: [https://coderooz.in](https://coderooz.in) +- Email: [contact@coderooz.in](mailto:contact@coderooz.in) +- Contact: [https://coderooz.in/contact](https://coderooz.in/contact?subject=[message_subject]&message=[message]) + +## Acknowledgments + +- Thanks to all contributors who help improve this project +- Built with the goal of reducing repetitive coding tasks + +## Support + +If you find this project helpful, consider supporting: + +- [Sponsor on GitHub](https://github.com/sponsors/Coderooz) +- [Contact for custom sponsorship](https://coderooz.in/contact?subject=Sponsorship) +- [Star this repository](https://github.com/coderooz/My_simple_functions) ⭐ + +--- + +

+ Made with ❀️ by CodeRooz +

diff --git a/Requester.py b/Requester.py index 599d715..c4e2604 100644 --- a/Requester.py +++ b/Requester.py @@ -1,5 +1,4 @@ import requests, random, time, websockets, aiohttp, asyncio -from requests import Response from urllib.parse import urlparse, parse_qsl, urlencode from FileHandler import read @@ -8,7 +7,7 @@ class Requester: Requester() =========== - Requester is a class for making HTTP & HTTPS requests easier specialy dureing the time of development. + Requester is a class for making HTTP & HTTPS requests easier speciall dureing the time of development. """ @@ -41,6 +40,9 @@ def __init__(self, agent:list=[], header:dict={}, proxy:list=[], ref:list=[], re if set_header: self.header = header + # if self._check_connection()==False: + # raise ConnectionRefusedError('There is some issues with the internet Connection. Please the internet connection before performing any requests.') + def get_proxy(self): """ This method gives a proxy url randomly. @@ -115,14 +117,13 @@ def parse_url_parameters(self, url:str): """ return self.get_urlinfo(url)['params'] - def request(self, url, method='get', params=None, data=None, json:dict={}, header:dict={}, cookies=None, timeout=5, redirect=True, verify=True, proxy=None, ref:str='', agent:str='', break_pt:list=[], setHeader:bool=False, response_code:int=200, raw:bool=False): + def request(self, url, method='get', params=None, data=None, json:dict={}, header:dict={}, cookies=None, timeout=5, redirect=True, verify=True, proxy=None, ref:str='', agent:str='', break_pt:list=[], setHeader:bool=False): break_pt = self.break_pt if break_pt is [] else break_pt if break_pt != []: time.sleep(random.uniform(break_pt[0], break_pt[1])) proxy = self.get_proxy() if proxy is None else proxy header = None if header=={} else self.headers(agent, ref, header, setHeader) - ret:Response if method.lower() == 'get': ret = requests.get(url, params=params, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore elif method.lower() == 'post': @@ -133,18 +134,9 @@ def request(self, url, method='get', params=None, data=None, json:dict={}, heade ret = requests.patch(url, params=params, data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore elif method.lower() == 'delete': ret = requests.delete(url,params=params,data=data, json=json, headers=header, cookies=cookies, timeout=timeout, allow_redirects=redirect, verify=verify, proxies=proxy) #type:ignore - ret.raise_for_status() - if raw == False and response_code==ret.status_code: - try: - ret = ret.json() - except: - try: - ret = ret.text - except: - ret = None - return ret - - def requestSessions(self, url:str, method:str='get', params=None, data=None, json=None, header=None, cookies=None, timeout:int=5, sessions=None, redirect=True, verify=True, proxy=None, ref:str='', agent:str='', pre_request:bool=False, break_pt:list=[], response_code:int=200, raw:bool=False): + return ret #type:ignore + + def requestSessions(self, url:str, method:str='get', params=None, data=None, json=None, header=None, cookies=None, timeout:int=5, sessions=None, redirect=True, verify=True, proxy=None, ref:str='', agent:str='', pre_request:bool=False, break_pt:list=[]): """ This method (requestSessions) is do make request based on the sessions. @@ -165,8 +157,6 @@ def requestSessions(self, url:str, method:str='get', params=None, data=None, jso agent (_type_, optional): Takes the user-agent detials. Defaults to None. pre_request (bool, optional): This parameter is responsible for adding sessions to the requsted url if given `True`. Defaults to False. break_pt (_type_, optional): _description_. Defaults to None. - - response_code (__type__, int): ... - - raw (__type__, bool): ... Returns: resonse: Returns the response of the requested url. @@ -202,15 +192,6 @@ def requestSessions(self, url:str, method:str='get', params=None, data=None, jso ret = s.patch(url, params=params, data=data, json=json) elif method.lower() == 'delete': ret = s.delete(url,params=params,data=data, json=json) - ret.raise_for_status() - if raw == False and response_code==ret.status_code: - try: - ret = ret.json() - except: - try: - ret = ret.text - except: - ret = None return ret, s def start_up(self): diff --git a/SECURITY.md b/SECURITY.md index e8c6ddd..4496fee 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,33 +2,62 @@ ## Supported Versions +The following versions of the project are currently being supported with security updates. + | Version | Supported | | ------- | ------------------ | -| 1.0.x | :white_check_mark: | +| 0.0.x | :white_check_mark: | ## Reporting a Vulnerability -If you discover a security vulnerability within Custom Functions, please send an email to Ranit Saha at contact@coderooz.in. All security vulnerabilities will be promptly addressed. +We take the security of this project seriously. If you discover a security vulnerability, please follow the responsible disclosure process below. + +### How to Report + +**DO NOT** open a public GitHub issue for security vulnerabilities. + +Instead, please report security vulnerabilities through one of these methods: + +1. **Email**: [contact@coderooz.in](mailto:contact@coderooz.in) +2. **Contact Form**: [CodeRooz Contact](https://coderooz.in/contact?subject=[Security_Vulnerability_Report]) Please include the following information in your report: -- Type of vulnerability -- Steps to reproduce -- Potential impact -- Suggested fix (if any) +- Description of the vulnerability +- Steps to reproduce the issue +- Potential impact of the vulnerability +- Any suggested fixes (if you have them) + +### What to Expect + +- **Acknowledgment**: You will receive an acknowledgment of your report within **48 hours** +- **Initial Assessment**: We will provide an initial assessment within **7 days** +- **Updates**: We will keep you informed of our progress throughout the process +- **Resolution**: We aim to resolve critical vulnerabilities within **30 days** + +### Security Best Practices for Users + +When using this package: + +1. **Keep Updated**: Always use the latest version of the package +2. **Review Dependencies**: Regularly review and update your project dependencies +3. **Input Validation**: Always validate and sanitize user inputs before passing them to any functions +4. **Secure Credentials**: Never hardcode sensitive information like API keys, passwords, or database credentials +5. **Database Security**: When using `DbHandler`, ensure your database files have appropriate file permissions + +### Security Measures in This Project -## Security Best Practices +- Input validation on all public methods +- Parameterized SQL queries to prevent SQL injection +- Secure file handling practices +- Regular dependency updates via Dependabot +- Automated security scanning in CI/CD pipelines -When using this library: +### Bug Bounty -1. Never hardcode database credentials -2. Use environment variables for sensitive configuration -3. Validate all user inputs -4. Use parameterized queries to prevent SQL injection -5. Keep dependencies up to date +Currently, this project does not offer a bug bounty program. However, we greatly appreciate responsible disclosure and will credit reporters in our security advisories (unless they prefer to remain anonymous). -## Response Timeline +--- -- Acknowledgment: Within 48 hours -- Initial assessment: Within 1 week -- Fix or mitigation: Depends on severity +**Author**: Ranit Saha +**Website**: [https://coderooz.in](https://coderooz.in) diff --git a/__init__.py b/__init__.py index e69de29..f0d6a13 100644 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1 @@ +all = ['DataHandler', 'Dbhandler', 'FileHandler', 'Requester'] \ No newline at end of file diff --git a/custom-functions.project-mcp.json b/custom-functions.project-mcp.json new file mode 100644 index 0000000..74899c9 --- /dev/null +++ b/custom-functions.project-mcp.json @@ -0,0 +1,144 @@ +{ + "project": { + "name": "custom-functions", + "scope": "project", + "environment": "development" + }, + "connection": { + "strategy": "runtime-first", + "preferredPortRange": [ + 47000, + 47020 + ], + "fallbackPorts": [ + 47000, + 47001, + 47005, + 47010 + ], + "retry": { + "maxRetries": 5, + "backoff": "exponential", + "baseDelay": 200, + "maxDelay": 3200 + }, + "timeout": 30000, + "maxConcurrentRequests": 10, + "healthCheck": { + "enabled": true, + "timeout": 5000, + "retries": 3, + "interval": 30000 + } + }, + "agent": { + "defaultAgent": "mcp-orchestrator", + "autoRegister": true, + "heartbeatInterval": 30000, + "permissions": { + "allowToolExecution": true + }, + "behavior": { + "autoOptimize": true, + "allowSelfModification": false, + "maxRetainedContexts": 100, + "autoMemoryCleanup": true + } + }, + "behavior": { + "ignore": [ + "node_modules/", + ".git/", + "logs/", + "dist/", + "build/", + ".next/", + "coverage/", + "*.log" + ], + "askBefore": [ + "destructive_actions" + ], + "autoApprove": [ + "read_operations" + ] + }, + "features": { + "multiAgent": true, + "chat": true, + "messaging": true, + "feedback": true, + "emulator": true, + "memory": { + "enabled": true, + "optimization": true, + "versioning": true + }, + "tasks": { + "concurrency": "atomic", + "retry": true, + "maxRetries": 3 + } + }, + "security": { + "inputSanitization": true, + "agentIsolation": true, + "idempotency": true + }, + "logging": { + "level": "minimal", + "errorsOnly": true, + "enabled": true + }, + "rules": { + "ignore": [ + "node_modules/", + ".git/", + "dist/", + "build/", + ".next/", + "coverage/", + "*.log" + ], + "protected": [ + "core/", + "infrastructure/" + ], + "scanExtensions": [ + ".js", + ".ts", + ".jsx", + ".tsx", + ".json", + ".md", + ".yml", + ".yaml" + ] + }, + "execution": { + "parallelAgents": true, + "maxAgents": 10, + "agentTimeout": 60000 + }, + "setup": { + "autoConfigure": true, + "autoHeal": true, + "maxSetupAttempts": 3, + "requiredEnvVars": [ + "MCP_SCOPE", + "MCP_PROJECT" + ], + "optionalEnvVars": [ + "MCP_AGENT", + "NODE_ENV", + "LOG_LEVEL" + ] + }, + "plugins": { + "emulator": { + "enabled": true, + "autoDetect": true + } + }, + "policies": {} +} diff --git a/pyproject.toml b/pyproject.toml index 02874d8..d9d89a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,76 +1,152 @@ [build-system] -requires = ["setuptools>=61.0", "wheel"] -build-backend = "setuptools.backends._legacy:_Backend" +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" [project] -name = "custom-functions" -version = "1.0.0" -description = "A collection of pre-made Python utilities for everyday tasks" -readme = "Readme.md" +name = "custom_functions" +version = "0.0.1" +description = "Pre-made Python functions for everyday tasks" +readme = "README.md" license = {text = "MIT"} authors = [ - {name = "Ranit Saha", email = "contact@coderooz.in"} + {name = "Ranit Saha", email = "contact@coderooz.in"}, +] +maintainers = [ + {name = "Ranit Saha", email = "contact@coderooz.in"}, +] +keywords = [ + "python", + "file-handling", + "requests", + "sqlite", + "database", + "data-handling", + "utilities", + "helper-functions", ] -requires-python = ">=3.12" classifiers = [ - "Development Status :: 4 - Beta", + "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Utilities", ] +requires-python = ">=3.12" dependencies = [ "pytz>=2023.3.0", "requests>=2.31.0", "db-sqlite3>=0.0.1", "pandas>=2.1.3", - "mysql-connector-python>=2.2.9", + "mysql-connector-python>=9.7.0", "openpyxl>=3.1.2", "PyPDF2>=3.0.0", ] [project.optional-dependencies] dev = [ - "pytest>=7.0.0", - "pytest-cov>=4.0.0", - "black>=23.0.0", - "isort>=5.0.0", - "flake8>=6.0.0", - "mypy>=1.0.0", - "twine>=4.0.0", + "pytest>=7.0", + "pytest-cov>=4.0", + "twine>=4.0.2", + "black>=23.0", + "isort>=5.0", + "flake8>=6.0", + "mypy>=1.0", ] [project.urls] Homepage = "https://github.com/coderooz/My_simple_functions" Repository = "https://github.com/coderooz/My_simple_functions" -Issues = "https://github.com/coderooz/My_simple_functions/issues" +"Bug Tracker" = "https://github.com/coderooz/My_simple_functions/issues" Changelog = "https://github.com/coderooz/My_simple_functions/blob/main/CHANGELOG.md" +Documentation = "https://github.com/coderooz/My_simple_functions#readme" +"Author Website" = "https://coderooz.in" -[project.scripts] -# Add CLI entry points here if needed +[tool.setuptools] +packages = ["custom_functions"] -[tool.setuptools.packages.find] -include = ["DataHandlers*", "DbHandler*", "FileHandler*", "Requester*", "HtmlScraper*"] +[tool.setuptools.package-dir] +"" = "." [tool.black] -line-length = 88 -target-version = ["py312", "py313"] +line-length = 120 +target-version = ['py312'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist + | __pycache__ +)/ +''' [tool.isort] profile = "black" -line_length = 88 +line_length = 120 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true -[tool.mypy] -python_version = "3.12" -warn_return_any = true -warn_unused_configs = true -disallow_untyped_defs = true +[tool.flake8] +max-line-length = 120 +max-complexity = 10 +exclude = [ + ".git", + "__pycache__", + "build", + "dist", + "*.egg-info", + ".eggs", + ".tox", + ".venv", + "venv", +] +per-file-ignores = [ + "__init__.py:F401,F403", +] [tool.pytest.ini_options] testpaths = ["tests"] -python_files = ["test_*.py"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] python_functions = ["test_*"] addopts = "-v --tb=short" + +[tool.mypy] +python_version = "3.12" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true + +[tool.coverage.run] +source = ["custom_functions"] +omit = [ + "tests/*", + "setup.py", + "__init__.py", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "pass", + "raise ImportError", +] +show_missing = true +fail_under = 0 diff --git a/requirement.txt b/requirement.txt deleted file mode 100644 index a2c609a..0000000 --- a/requirement.txt +++ /dev/null @@ -1,11 +0,0 @@ -requests -sqlite3 -pandas -mysql -datetime -json -csv -os -openpyxl -PyPDF2 - diff --git a/setup.py b/setup.py index 1056261..7a403a3 100644 --- a/setup.py +++ b/setup.py @@ -21,15 +21,11 @@ long_description_content_type="text/markdown", long_description=long_description, packages=find_packages(), - install_requires=['time', 'json', 'os', 'pytz', 're','requests', 'datetime','json', 'csv', 'sqlite3', 'pandas', 'json',' mysql-connector-python', 'openpyxl','PyPDF2'], + znstall_requires=['pytz>=2023.3.0','requests>=2.31.0', 'db-sqlite3>=0.0.1', 'pandas>=2.1.3',' mysql-connector-python>=2.2.9', 'openpyxl>=3.1.2','PyPDF2>=3.0.0'], keywords=['python', 'file', 'fileHandleing', 'requests', 'requestsHandling', 'sqlite', 'sqlDbHandling', 'mysql'], - classifiers=[ - "Development Status :: 1 - Planning", - "Intended Audience :: Developers", - "Programming Language :: Python :: 3", - "Operating System :: Unix", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows", - ] - ext_modules=cythonize(["DataHandler.pyx", "DbHandler.pyx", ""]) + classifiers=["License :: OSI Approved :: MIT Licence","Development Status :: 1 - Planning","Intended Audience :: Developers", + "Programming Language :: Python :: 3","Operating System :: Unix","Operating System :: MacOS :: MacOS X","Operating System :: Microsoft :: Windows"], + licence = "MIT", + extras_require={"dev": ["pytest>=7.0", "twine>=4.0.2"]}, + python_requires=">=3.12.0" ) \ No newline at end of file