Merge pull request #5 from mutairibassam/fixes/thread-not-stored

Fixes/thread not stored

.gitignore

@@ -2,4 +2,181 @@
 sample/
 config.yml
 *.txt
-.vscode
+.vscode
+
+# Created by https://www.toptal.com/developers/gitignore/api/python
+# Edit at https://www.toptal.com/developers/gitignore?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+# End of https://www.toptal.com/developers/gitignore/api/python

Dockerfile

@@ -6,4 +6,4 @@ COPY . .
 
 RUN pip install -r requirements.txt
 
-CMD [ "python", "app.py" ]
+CMD [ "python", "-m", "biz"]

README.md

@@ -30,7 +30,7 @@ docker run biz
 ```bash
 git clone https://github.com/mutairibassam/biz.git
 cd biz
-python app.py
+python3 -m biz
 ```
 
 ### Add your secrets

biz/apis/api.py

@@ -9,7 +9,6 @@
 def send(
         thread_id: str,
         subject: str,
-        image_url: str,
         thread_comment: str
     ) -> None:
 
@@ -46,9 +45,9 @@ def send(
     /// To send the data you just need to do simple GET request
 
     """
-    archiver = f'https://archived.moe/biz/thread/{thread_id}'
+    archiver: str = f'https://archived.moe/biz/thread/{thread_id}'
 
-    message = f"Missing id: {thread_id}\nSubject: {subject}\nComment: {thread_comment}\n{image_url}\nArchiver: {archiver}\nv1.0"
+    message: str = f"Missing id: {thread_id}\nSubject: {subject}\nComment: {thread_comment}\nArchiver: {archiver}\nv1.1"
     try:
         telegram_url: str = f'https://api.telegram.org/bot{token}/sendMessage?chat_id={chat_id}&text={message}'
         r.get(telegram_url,timeout=5)
@@ -66,8 +65,12 @@ def deleted(ids: list) -> list:
 
     """
     url: str = 'https://boards.4channel.org/biz/thread/'
-    # deleted_ids: list = [i for i in ids if r.get(url + str(i), timeout=5).status_code == 404]
 
+    # list comprehension logic is commented because we don't use list comprehension for readability purposes
+    # and since the array size will always be the same length we compromise
+    # code readability over performance.
+
+    # deleted_ids: list = [i for i in ids if r.get(url + str(i), timeout=5).status_code == 404]
     not_found_ids: list = []
     for i in ids:
         try:
@@ -79,7 +82,7 @@ def deleted(ids: list) -> list:
             continue
     return not_found_ids
 
-def get_threads(board: Board) -> list[Board]:
+def get_threads(board: Board) -> list:
     """ return all threads with its data """
     try:
         threads: list = board.get_all_threads()
@@ -88,7 +91,7 @@ def get_threads(board: Board) -> list[Board]:
         print(err)
         return err
 
-def iterate(deleted_ids: list):
+def iterate(deleted_ids: list) -> None:
     """
     deleted_ids  ->  confirmed deleted ids
 
@@ -106,14 +109,18 @@ def iterate(deleted_ids: list):
     function.
 
     """
+    print(f"deleted ids {deleted_ids}")
     local_data = read()
     for eachline in local_data:
         try:
-            thread_id, subject, image_url, comment = eachline.split("~")
+            # _ variable is thread image url but it's been removed
+            # since it might have inappropriate images
+            thread_id, subject, _, *comment = eachline.split("~")
             if int(thread_id) in deleted_ids:
-                print(f'Missing id: {thread_id}')
-                send(thread_id, subject, image_url, comment)
+                print(f'thread to be sent: {thread_id}')
+                send(thread_id, subject, comment)
         except ValueError as err:
+            print(eachline)
             print(err)
             continue
 

biz/biz_app.py

@@ -2,7 +2,7 @@
 import time
 from biz.board import Board
 from biz.config import write, compare
-from biz.topic import prepare
+from biz.topic import prepare, dump
 from biz.apis.api import deleted, get_threads, iterate
 
 def biz_app() -> None:
@@ -11,18 +11,19 @@ def biz_app() -> None:
     new_ids: list = []
     prev_ids: list = []
     while True:
-        setup()
-        threads = get_threads(board=board)
-        new_ids = prepare(threads)
+        time.sleep(15)
+        threads: list = get_threads(board=board)
+        # we shouldn't update the storage.txt until we compare.
+        new_ids: list = prepare(threads)
 
         diff: list = compare(new_ids, prev_ids)
         prev_ids = new_ids
         deleted_ids: list = deleted(diff)
         if(len(deleted_ids) > 0):
             iterate(deleted_ids)
+        # here we should update storage.txt with the new threads
+        # clean and dump
+        write("")
+        dump(threads)
         new_ids = []
-
-def setup():
-    """docs"""
-    time.sleep(10)
-    write("")
+

biz/config.py

@@ -1,6 +1,8 @@
 """docs"""
 from yaml import full_load, YAMLError
 
+file_path: str = "./storage.txt"
+
 def get_secret() -> dict:
     """ 
     get telegram secret keys from yml file 
@@ -32,19 +34,17 @@ def get_secret() -> dict:
         except YAMLError as exception:
             raise exception
 
-file_path: str = "./storage.txt"
-
-def write(content):
+def write(content) -> None:
     """ replace all existing file data will the new data """
     with open(file_path, "w", encoding="utf-8") as file:
         file.write(content)
 
-def append(content):
+def append(content) -> None:
     """ add more the existing data """
     with open(file_path, "a",encoding="utf-8") as file:
         file.write(content)
 
-def read():
+def read() -> list:
     """ read current file data and return each line """
     with open(file_path, "r",encoding="utf-8") as file:
         lines = file.readlines()

biz/topic.py

@@ -24,7 +24,22 @@ def __init__(self, thread_id, subject, image_url, comment):
         append(thread)
 
 
-def prepare(threads):
+def prepare(threads) -> list:
+    """
+    threads are all the threads that we receive from Board API to be stored
+    locally for future reference.
+
+    Since we need to compare between previous and new ids the below code
+    return thread new ids.
+    """
+    thread_ids = []
+    for thread in threads:
+        thread_id = thread.id
+        thread_ids.append(thread_id)
+
+    return thread_ids
+
+def dump(threads) -> None:
     """
     threads are all the threads that we receive from Board API to be stored
     locally for future reference.
@@ -34,14 +49,22 @@ def prepare(threads):
     threads.
     
     Now we diff among threads if there is a new line.
+    
+    Threads structure:
+    threads {
+        id 
+        topic {
+            title
+            comment
+            thumbnail
+        }
+    }
     """
-    thread_ids = []
     for thread in threads:
         thread_id = thread.id
-        thread_ids.append(thread_id)
 
         topic = thread.topic
         clean_comment = re.sub(r'[^a-zA-Z0-9\s]+', '', topic.text_comment).replace("\n", "").replace("~","")
+        clean_title = str(topic.subject).replace("~","")
 
-        Topic(thread_id, topic.subject, topic.thumbnail_url, clean_comment)
-    return thread_ids
+        Topic(thread_id, clean_title, topic.thumbnail_url, clean_comment)